diff --git a/pyproject.toml b/pyproject.toml index 32ed476929..d92f5d6759 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,22 @@ sbcli-dev = "simplyblock_cli.cli:main" simplyblock-service = "simplyblock_core.services.__main__:main" simplyblock-task-runner = "simplyblock_core.services.task_runners:main" +# Storage-array plugins of the multi-attach array LUN mode +# (simplyblock_core.array); the name is the value of Cluster.array_plugin. +[project.entry-points."simplyblock.array_plugins"] +aws-ebs = "simplyblock_core.array.aws_ebs:AwsEbsPlugin" +pure-flasharray = "simplyblock_core.array.san:PureFlashArrayPlugin" +netapp-ontap = "simplyblock_core.array.san:NetAppOntapPlugin" +dell-powerstore = "simplyblock_core.array.san:DellPowerStorePlugin" +dell-powermax = "simplyblock_core.array.san:DellPowerMaxPlugin" +dell-powerflex = "simplyblock_core.array.san:DellPowerFlexPlugin" +hpe-wsapi = "simplyblock_core.array.san:HpeWsapiPlugin" +hpe-nimble = "simplyblock_core.array.san:HpeNimblePlugin" +ibm-flashsystem = "simplyblock_core.array.san:IbmFlashSystemPlugin" +hitachi-vsp = "simplyblock_core.array.san:HitachiVspPlugin" +huawei-dorado = "simplyblock_core.array.san:HuaweiDoradoPlugin" +infinidat-infinibox = "simplyblock_core.array.san:InfinidatInfiniBoxPlugin" + [dependency-groups] test = [ "mock", diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 2b907415aa..d99c2b1c36 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -115,6 +115,16 @@ commands: required: false type: str default: "" + - name: "--lblk-journal-only" + help: "With --lblk on a multi-attach cluster whose array plugin creates each node's data LUN at add-node and keeps its journal locally: the one selected disk or partition is the node's journal and nothing else. It is used whole (a disk is never partitioned), and the node has no local data device; its data device is the array LUN. Needs exactly one disk or partition, named with --blk-names or --blk-serials." + dest: lblk_journal_only + type: bool + action: store_true + - name: "--lblk-array-only" + help: "With --lblk on a multi-attach cluster whose array plugin creates each node's LUNs at add-node and keeps its journal on a LUN: the node uses no local device. Its journal LUN and data LUN are created on the array when the node is added. Not with --blk-names, --blk-names-exclude, --blk-serials, --lblk-journal-only or --lblk-journal-device." + dest: lblk_array_only + type: bool + action: store_true - name: "--lblk-journal-device" help: "With --lblk and whole disks: dedicate each node's smallest disk to the journal instead of partitioning it into a journal partition and a data partition (the default). Needs at least two disks per node. Partition selections always carry their journal in a split partition." dest: lblk_journal_device @@ -578,19 +588,6 @@ commands: dest: force type: bool action: store_true - - name: resize-device - help: "Grows a Linux block device (lblk) online to the size of its block device." - usage: > - Re-reads the size of the device's block device on the storage node's host, for example, after its LUN was - grown on the storage array, and grows the storage device online without interrupting I/O. Shrinking is - refused. The placement weight of the device keeps its original size, so the added capacity is only fully - usable once all data devices of the cluster have grown by the same factor, or after cluster grow-devices - has updated the weights of unevenly grown devices and rebalanced the cluster. - arguments: - - name: "device_id" - help: "The storage device id." - dest: device_id - type: str - name: add-device help: "Adds a new storage device." usage: > @@ -1040,11 +1037,47 @@ commands: private: true - name: "--strict-node-anti-affinity" help: > - Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. This - requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster. + Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. The + cluster needs the same node floor as every cluster (_data-chunks-per-stripe + 2 x parity-chunks-per-stripe_ + nodes); the flag only affects status reporting. dest: strict_node_anti_affinity type: bool action: store_true + - name: "--raid-devices" + help: > + Declare that every data device is protected below simplyblock (hardware RAID LUN, md or dm RAID). Lowers + the data-device floor from n+2k to n+k and allows a stripe to place several chunks on one device when no + other device is writable. Cannot be changed after cluster creation. + dest: raid_devices + type: bool + action: store_true + - name: "--multi-attach" + help: > + Multi-attach array LUN mode. Every data device is a LUN of an external array that already protects it + and presents it to several storage nodes. The cluster runs 1+0 erasure coding (the data-chunks and + parity-chunks settings are overridden); on a node failure the LUN is reopened on a standby node and + reached over an additional NVMe-oF path. Requires --raid-devices, --ftt and --ha-type ha. Cannot be + changed after cluster creation. + dest: multi_attach + type: bool + action: store_true + - name: "--multi-attach-warm-standby" + help: > + Multi-attach only: keep a warm standby instance of every LUN on each node it is attached to but not + served by. A takeover then activates it on the first I/O instead of loading the LUN's page map cold + (minutes on large LUNs). Each standby holds the LUN's page map in hugepages on its node. + dest: multi_attach_warm_standby + type: bool + action: store_true + - name: "--ftt" + help: > + Node failures to tolerate in multi-attach mode (1 or 2; 2 adds a tertiary LVS node and needs at least + 3 storage nodes). Required with --multi-attach, refused without it. + dest: ftt + type: int + choices: + - 1 + - 2 - name: "--enable-failure-domain" help: "Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed." dest: enable_failure_domain @@ -1327,11 +1360,47 @@ commands: private: true - name: "--strict-node-anti-affinity" help: > - Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. - This requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster." + Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. The + cluster needs the same node floor as every cluster (_data-chunks-per-stripe + 2 x parity-chunks-per-stripe_ + nodes); the flag only affects status reporting. dest: strict_node_anti_affinity type: bool action: store_true + - name: "--raid-devices" + help: > + Declare that every data device is protected below simplyblock (hardware RAID LUN, md or dm RAID). Lowers + the data-device floor from n+2k to n+k and allows a stripe to place several chunks on one device when no + other device is writable. Cannot be changed after cluster creation. + dest: raid_devices + type: bool + action: store_true + - name: "--multi-attach" + help: > + Multi-attach array LUN mode. Every data device is a LUN of an external array that already protects it + and presents it to several storage nodes. The cluster runs 1+0 erasure coding (the data-chunks and + parity-chunks settings are overridden); on a node failure the LUN is reopened on a standby node and + reached over an additional NVMe-oF path. Requires --raid-devices, --ftt and --ha-type ha. Cannot be + changed after cluster creation. + dest: multi_attach + type: bool + action: store_true + - name: "--multi-attach-warm-standby" + help: > + Multi-attach only: keep a warm standby instance of every LUN on each node it is attached to but not + served by. A takeover then activates it on the first I/O instead of loading the LUN's page map cold + (minutes on large LUNs). Each standby holds the LUN's page map in hugepages on its node. + dest: multi_attach_warm_standby + type: bool + action: store_true + - name: "--ftt" + help: > + Node failures to tolerate in multi-attach mode (1 or 2; 2 adds a tertiary LVS node and needs at least + 3 storage nodes). Required with --multi-attach, refused without it. + dest: ftt + type: int + choices: + - 1 + - 2 - name: "--enable-failure-domain" help: "Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed." dest: enable_failure_domain @@ -1523,23 +1592,126 @@ commands: type: str completer: _completer_get_cluster_list - name: grow-devices - help: "Grows every Linux block device (lblk) of the cluster online to the size of its block device." + help: "Grows all Linux block devices (lblk) of the cluster online by the same factor." usage: > - Re-reads the size of every online data device of an lblk cluster and grows the devices whose block device - has grown, one at a time. Used after the LUNs of the cluster were grown on the storage array. When all - devices have grown by the same factor, nothing else changes. When they have grown unevenly, the device - weights are set to the new sizes and the cluster is rebalanced, which re-places a large part of the data - in the background; this needs an active cluster with all nodes and devices online and no running - migration. Grow all LUNs on the array first, then run this command once. + Re-reads the size of the block device of every data device of an lblk cluster, for example, after the + LUNs were grown on the storage array, and grows every device online by the same factor: the smallest + growth of any of the block devices. With three LUNs grown by 1.5 and three by 2.0, every device grows by + 1.5 and the rest stays unused. If one block device did not grow, nothing grows. Device weights and data + placement never change. All data devices and their nodes must be online. A block device smaller than its + device is refused and raises an alert. Grow all LUNs on the array first, then run this command once. arguments: - name: "cluster_id" help: "The cluster id." dest: cluster_id type: str completer: _completer_get_cluster_list - - name: "--no-rebalance" - help: "Keep the device weights after an uneven growth and do not rebalance the cluster." - dest: no_rebalance + - name: "--lun-factor" + help: > + Multi-attach cluster with a registered array only: first grow every LUN the array plugin created by + this factor (greater than 1, e.g. 1.5) at the array and wait until every node sees the new size. + dest: lun_factor + type: str + - name: "--lun-size" + help: > + Multi-attach cluster with a registered array only, instead of --lun-factor: first grow every LUN the + array plugin created to this size (e.g. `6T`); a LUN already that large is left alone, so the command + can be repeated after a failure. Nodes added later get LUNs of this size. + dest: lun_size + type: size + default: "0" + - name: set-array + help: "Registers the storage array of a multi-attach cluster (or updates it, e.g. to rotate credentials)." + usage: > + simplyblock creates, multi-attaches, grows, fences and deletes the LUNs of a multi-attach cluster through + the array plugin registered here. The plugin is loaded and connected with the credentials before anything + is saved. The credentials are wrapped by the cluster's KMS (or stay in an external Vault KV / KMIP secret named + by --credential-ref), never stored in the clear. With --lun-size, + every node added afterwards gets a LUN of that size created and attached on the array. Its journal is + kept on a local disk or partition of its host (--journal local, the default: configure the host with + --lblk-journal-only) or on a LUN of its own created next to it (--journal lun: configure the host with + --lblk-array-only), for example in a pool of higher performance named by --journal-options. + arguments: + - name: "cluster_id" + help: "The cluster id." + dest: cluster_id + type: str + completer: _completer_get_cluster_list + - name: "--plugin" + help: "The array plugin, e.g. `aws-ebs`." + dest: plugin + type: str + required: true + - name: "--endpoint" + help: "The array's management endpoint (for aws-ebs: an EC2 endpoint URL override, usually empty)." + dest: endpoint + type: str + default: "" + - name: "--options" + help: 'Plugin options as a JSON object, e.g. `{"az": "us-east-1a", "iops": 6000}`.' + dest: options + type: str + - name: "--credentials-file" + help: > + A JSON file with the array credentials as an object of strings (for aws-ebs: aws_access_key_id, + aws_secret_access_key; omit to use the instance role). + dest: credentials_file + type: str + - name: "--credential-ref" + help: > + Instead of --credentials-file: a secret kept outside simplyblock, as JSON, e.g. + `{"kind": "vault-kv", "mount": "secret", "path": "arrays/pure01"}` or `{"kind": "kmip-secret", "uid": "42"}`. + dest: credential_ref + type: str + - name: "--lun-size" + help: "Size of the LUN created for each node added to the cluster, e.g. `1T`. Default: none." + dest: lun_size + type: size + default: "0" + - name: "--journal" + help: > + Where each node's journal is kept: `local` (a disk or partition of its host) or `lun` (a LUN of its own, + created on the array next to the node's data LUN; needs --lun-size). Default: `local`. + dest: journal + type: str + default: "local" + choices: + - "local" + - "lun" + - name: "--journal-lun-size" + help: "With --journal lun: size of each node's journal LUN, e.g. `64G`. Default: 3% of --lun-size, at least 2 GiB." + dest: journal_lun_size + type: size + default: "0" + - name: "--journal-options" + help: > + With --journal lun: plugin options for the journal LUNs as a JSON object, applied over --options, e.g. + `{"pool": "fast-pool"}` to create them in a pool of higher performance or priority. + dest: journal_options + type: str + - name: get-array + help: "Shows the storage array registered for a multi-attach cluster (without its credentials)." + arguments: + - name: "cluster_id" + help: "The cluster id." + dest: cluster_id + type: str + completer: _completer_get_cluster_list + - name: remove-array + help: "Removes the storage array registration of a multi-attach cluster and its credentials." + arguments: + - name: "cluster_id" + help: "The cluster id." + dest: cluster_id + type: str + completer: _completer_get_cluster_list + - name: "array_id" + help: "The array id." + dest: array_id + type: str + - name: "--force" + help: "Remove it even though LUNs created through it exist (they can then no longer be fenced)." + dest: force type: bool default: false action: store_true diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 0d21c5fcb1..2989a04be7 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -60,7 +60,6 @@ def init_storage_node(self): self.init_storage_node__device_hang(subparser) self.init_storage_node__get_device(subparser) self.init_storage_node__restart_device(subparser) - self.init_storage_node__resize_device(subparser) self.init_storage_node__add_device(subparser) self.init_storage_node__remove_device(subparser) self.init_storage_node__set_failed_device(subparser) @@ -113,6 +112,8 @@ def init_storage_node__configure(self, subparser): subcommand.add_argument('--blk-names', help='Comma separated list of block devices to use (requires --lblk). Each entry is a persistent /dev/disk name, such as /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi0 or /dev/disk/by-partuuid/28427de0-1916-4c05-895f-0829cd8790ba. A kernel name (sdb) or kernel path (/dev/sdb) is refused: it states a position in this boot\'s enumeration order, and it would name another disk after a reboot that probes the controllers in another order. The path is taken as given: the node records it, and add-node and every restart open the device by that path alone, never by serial or kernel name. A path that no longer exists is a missing device. Use --blk-serials for a device udev published no persistent name for. Requested devices must be eligible; a busy device is an error.', type=str, default='', dest='blk_names', required=False) subcommand.add_argument('--blk-names-exclude', help='Comma separated list of block devices to exclude (requires --lblk), spelled the way --blk-names takes them. All other eligible disks are used.', type=str, default='', dest='blk_names_exclude', required=False) subcommand.add_argument('--blk-serials', help='Comma separated list of block device serial numbers (or WWNs) to use (requires --lblk).', type=str, default='', dest='blk_serials', required=False) + subcommand.add_argument('--lblk-journal-only', help='With --lblk on a multi-attach cluster whose array plugin creates each node\'s data LUN at add-node and keeps its journal locally: the one selected disk or partition is the node\'s journal and nothing else. It is used whole (a disk is never partitioned), and the node has no local data device; its data device is the array LUN. Needs exactly one disk or partition, named with --blk-names or --blk-serials.', dest='lblk_journal_only', action='store_true') + subcommand.add_argument('--lblk-array-only', help='With --lblk on a multi-attach cluster whose array plugin creates each node\'s LUNs at add-node and keeps its journal on a LUN: the node uses no local device. Its journal LUN and data LUN are created on the array when the node is added. Not with --blk-names, --blk-names-exclude, --blk-serials, --lblk-journal-only or --lblk-journal-device.', dest='lblk_array_only', action='store_true') subcommand.add_argument('--lblk-journal-device', help='With --lblk and whole disks: dedicate each node\'s smallest disk to the journal instead of partitioning it into a journal partition and a data partition (the default). Needs at least two disks per node. Partition selections always carry their journal in a split partition.', dest='lblk_journal_device', action='store_true') subcommand.add_argument('--jm-percent', help='Journal size in percent of the node\'s total selected capacity when the journal is carved from a selected partition or disk (requires --lblk, not with --lblk-journal-device). Default: `3`.', type=int, default=3, dest='jm_percent', required=False) subcommand.add_argument('--force', help='Force format detected or passed nvme pci address to 4K and clean partitions. With --lblk: mark partitioned disks eligible; the partition wipe happens at add-node with --force-format.', dest='force', action='store_true') @@ -256,10 +257,6 @@ def init_storage_node__restart_device(self, subparser): subcommand.add_argument('device_id', help='The storage device id.', type=str) subcommand.add_argument('--force', help='Force restart.', dest='force', action='store_true') - def init_storage_node__resize_device(self, subparser): - subcommand = self.add_sub_command(subparser, 'resize-device', 'Grows a Linux block device (lblk) online to the size of its block device.') - subcommand.add_argument('device_id', help='The storage device id.', type=str) - def init_storage_node__add_device(self, subparser): subcommand = self.add_sub_command(subparser, 'add-device', 'Adds a new storage device.') subcommand.add_argument('device_id', help='The storage device id.', type=str) @@ -383,6 +380,9 @@ def init_cluster(self): self.init_cluster__status(subparser) self.init_cluster__complete_expand(subparser) self.init_cluster__grow_devices(subparser) + self.init_cluster__set_array(subparser) + self.init_cluster__get_array(subparser) + self.init_cluster__remove_array(subparser) self.init_cluster__show(subparser) self.init_cluster__get(subparser) if self.developer_mode: @@ -468,7 +468,11 @@ def init_cluster__create(self, subparser): subcommand.add_argument('--inflight-io-threshold', help='The number of inflight IOs allowed before the IO queuing starts. Default: `4`.', type=int, default=4, dest='inflight_io_threshold') if self.developer_mode: subcommand.add_argument('--disable-monitoring', help='Disable monitoring stack, false by default. Default: `false`.', dest='disable_monitoring', action='store_true') - subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. This requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster.', dest='strict_node_anti_affinity', action='store_true') + subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. The cluster needs the same node floor as every cluster (_data-chunks-per-stripe + 2 x parity-chunks-per-stripe_ nodes); the flag only affects status reporting.', dest='strict_node_anti_affinity', action='store_true') + subcommand.add_argument('--raid-devices', help='Declare that every data device is protected below simplyblock (hardware RAID LUN, md or dm RAID). Lowers the data-device floor from n+2k to n+k and allows a stripe to place several chunks on one device when no other device is writable. Cannot be changed after cluster creation.', dest='raid_devices', action='store_true') + subcommand.add_argument('--multi-attach', help='Multi-attach array LUN mode. Every data device is a LUN of an external array that already protects it and presents it to several storage nodes. The cluster runs 1+0 erasure coding (the data-chunks and parity-chunks settings are overridden); on a node failure the LUN is reopened on a standby node and reached over an additional NVMe-oF path. Requires --raid-devices, --ftt and --ha-type ha. Cannot be changed after cluster creation.', dest='multi_attach', action='store_true') + subcommand.add_argument('--multi-attach-warm-standby', help='Multi-attach only: keep a warm standby instance of every LUN on each node it is attached to but not served by. A takeover then activates it on the first I/O instead of loading the LUN\'s page map cold (minutes on large LUNs). Each standby holds the LUN\'s page map in hugepages on its node.', dest='multi_attach_warm_standby', action='store_true') + subcommand.add_argument('--ftt', help='Node failures to tolerate in multi-attach mode (1 or 2; 2 adds a tertiary LVS node and needs at least 3 storage nodes). Required with --multi-attach, refused without it.', type=int, dest='ftt', choices=[1,2,]) subcommand.add_argument('--enable-failure-domain', help='Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='enable_failure_domain', action='store_true') subcommand.add_argument('--disable-lvol-geometry', help='Create new logical volumes with ndcs=npcs=0 instead of the cluster\'s erasure coding geometry. Enabled by default; can be changed later with `cluster set-lvol-geometry`.', dest='disable_lvol_geometry', action='store_true') subcommand.add_argument('--sync-replication', help='Enable site-level synchronous replication over exactly two sites. Each storage node must then be added with --site; every lvstore is replicated to the other site. Requires --ha-type ha. Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='sync_replication', action='store_true') @@ -518,7 +522,11 @@ def init_cluster__add(self, subparser): subcommand.add_argument('--max-queue-size', help='The max size the queue will grow. Default: `128`.', type=int, default=128, dest='max_queue_size') if self.developer_mode: subcommand.add_argument('--inflight-io-threshold', help='The number of inflight IOs allowed before the IO queuing starts. Default: `4`.', type=int, default=4, dest='inflight_io_threshold') - subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. This requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster."', dest='strict_node_anti_affinity', action='store_true') + subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. The cluster needs the same node floor as every cluster (_data-chunks-per-stripe + 2 x parity-chunks-per-stripe_ nodes); the flag only affects status reporting.', dest='strict_node_anti_affinity', action='store_true') + subcommand.add_argument('--raid-devices', help='Declare that every data device is protected below simplyblock (hardware RAID LUN, md or dm RAID). Lowers the data-device floor from n+2k to n+k and allows a stripe to place several chunks on one device when no other device is writable. Cannot be changed after cluster creation.', dest='raid_devices', action='store_true') + subcommand.add_argument('--multi-attach', help='Multi-attach array LUN mode. Every data device is a LUN of an external array that already protects it and presents it to several storage nodes. The cluster runs 1+0 erasure coding (the data-chunks and parity-chunks settings are overridden); on a node failure the LUN is reopened on a standby node and reached over an additional NVMe-oF path. Requires --raid-devices, --ftt and --ha-type ha. Cannot be changed after cluster creation.', dest='multi_attach', action='store_true') + subcommand.add_argument('--multi-attach-warm-standby', help='Multi-attach only: keep a warm standby instance of every LUN on each node it is attached to but not served by. A takeover then activates it on the first I/O instead of loading the LUN\'s page map cold (minutes on large LUNs). Each standby holds the LUN\'s page map in hugepages on its node.', dest='multi_attach_warm_standby', action='store_true') + subcommand.add_argument('--ftt', help='Node failures to tolerate in multi-attach mode (1 or 2; 2 adds a tertiary LVS node and needs at least 3 storage nodes). Required with --multi-attach, refused without it.', type=int, dest='ftt', choices=[1,2,]) subcommand.add_argument('--enable-failure-domain', help='Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='enable_failure_domain', action='store_true') subcommand.add_argument('--disable-lvol-geometry', help='Create new logical volumes with ndcs=npcs=0 instead of the cluster\'s erasure coding geometry. Enabled by default; can be changed later with `cluster set-lvol-geometry`.', dest='disable_lvol_geometry', action='store_true') subcommand.add_argument('--sync-replication', help='Enable site-level synchronous replication over exactly two sites. Each storage node must then be added with --site; every lvstore is replicated to the other site. Requires --ha-type ha. Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='sync_replication', action='store_true') @@ -567,9 +575,33 @@ def init_cluster__complete_expand(self, subparser): subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list def init_cluster__grow_devices(self, subparser): - subcommand = self.add_sub_command(subparser, 'grow-devices', 'Grows every Linux block device (lblk) of the cluster online to the size of its block device.') + subcommand = self.add_sub_command(subparser, 'grow-devices', 'Grows all Linux block devices (lblk) of the cluster online by the same factor.') + subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list + subcommand.add_argument('--lun-factor', help='Multi-attach cluster with a registered array only: first grow every LUN the array plugin created by this factor (greater than 1, e.g. 1.5) at the array and wait until every node sees the new size.', type=str, dest='lun_factor') + subcommand.add_argument('--lun-size', help='Multi-attach cluster with a registered array only, instead of --lun-factor: first grow every LUN the array plugin created to this size (e.g. `6T`); a LUN already that large is left alone, so the command can be repeated after a failure. Nodes added later get LUNs of this size.', type=size_type(), default='0', dest='lun_size') + + def init_cluster__set_array(self, subparser): + subcommand = self.add_sub_command(subparser, 'set-array', 'Registers the storage array of a multi-attach cluster (or updates it, e.g. to rotate credentials).') + subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list + subcommand.add_argument('--plugin', help='The array plugin, e.g. `aws-ebs`.', type=str, dest='plugin', required=True) + subcommand.add_argument('--endpoint', help='The array\'s management endpoint (for aws-ebs: an EC2 endpoint URL override, usually empty).', type=str, default='', dest='endpoint') + subcommand.add_argument('--options', help='Plugin options as a JSON object, e.g. `{"az": "us-east-1a", "iops": 6000}`.', type=str, dest='options') + subcommand.add_argument('--credentials-file', help='A JSON file with the array credentials as an object of strings (for aws-ebs: aws_access_key_id, aws_secret_access_key; omit to use the instance role).', type=str, dest='credentials_file') + subcommand.add_argument('--credential-ref', help='Instead of --credentials-file: a secret kept outside simplyblock, as JSON, e.g. `{"kind": "vault-kv", "mount": "secret", "path": "arrays/pure01"}` or `{"kind": "kmip-secret", "uid": "42"}`.', type=str, dest='credential_ref') + subcommand.add_argument('--lun-size', help='Size of the LUN created for each node added to the cluster, e.g. `1T`. Default: none.', type=size_type(), default='0', dest='lun_size') + subcommand.add_argument('--journal', help='Where each node\'s journal is kept: `local` (a disk or partition of its host) or `lun` (a LUN of its own, created on the array next to the node\'s data LUN; needs --lun-size). Default: `local`.', type=str, default='local', dest='journal', choices=['local','lun',]) + subcommand.add_argument('--journal-lun-size', help='With --journal lun: size of each node\'s journal LUN, e.g. `64G`. Default: 3%% of --lun-size, at least 2 GiB.', type=size_type(), default='0', dest='journal_lun_size') + subcommand.add_argument('--journal-options', help='With --journal lun: plugin options for the journal LUNs as a JSON object, applied over --options, e.g. `{"pool": "fast-pool"}` to create them in a pool of higher performance or priority.', type=str, dest='journal_options') + + def init_cluster__get_array(self, subparser): + subcommand = self.add_sub_command(subparser, 'get-array', 'Shows the storage array registered for a multi-attach cluster (without its credentials).') + subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list + + def init_cluster__remove_array(self, subparser): + subcommand = self.add_sub_command(subparser, 'remove-array', 'Removes the storage array registration of a multi-attach cluster and its credentials.') subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list - subcommand.add_argument('--no-rebalance', help='Keep the device weights after an uneven growth and do not rebalance the cluster.', default=False, dest='no_rebalance', action='store_true') + subcommand.add_argument('array_id', help='The array id.', type=str) + subcommand.add_argument('--force', help='Remove it even though LUNs created through it exist (they can then no longer be fenced).', default=False, dest='force', action='store_true') def init_cluster__show(self, subparser): subcommand = self.add_sub_command(subparser, 'show', 'Shows a cluster\'s statistics.') @@ -1608,8 +1640,6 @@ def run(self): ret = self.storage_node__get_device(sub_command, args) elif sub_command in ['restart-device']: ret = self.storage_node__restart_device(sub_command, args) - elif sub_command in ['resize-device']: - ret = self.storage_node__resize_device(sub_command, args) elif sub_command in ['add-device']: ret = self.storage_node__add_device(sub_command, args) elif sub_command in ['remove-device']: @@ -1726,6 +1756,12 @@ def run(self): ret = self.cluster__complete_expand(sub_command, args) elif sub_command in ['grow-devices']: ret = self.cluster__grow_devices(sub_command, args) + elif sub_command in ['set-array']: + ret = self.cluster__set_array(sub_command, args) + elif sub_command in ['get-array']: + ret = self.cluster__get_array(sub_command, args) + elif sub_command in ['remove-array']: + ret = self.cluster__remove_array(sub_command, args) elif sub_command in ['show']: ret = self.cluster__show(sub_command, args) elif sub_command in ['get']: diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 3a9b470dc6..769a139007 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -8,11 +8,14 @@ from pathlib import Path from typing import Any +from pydantic import SecretStr + from simplyblock_cli.alerting_config_parser import parse_alerting_config from simplyblock_core import cluster_ops, constants, db_controller, utils from simplyblock_core import mgmt_node_ops as mgmt_ops from simplyblock_core import storage_node_ops as storage_ops from simplyblock_core.controllers import ( + array_controller, device_controller, fdb_backup_controller, health_controller, @@ -269,6 +272,23 @@ def storage_node__configure(self, sub_command, args): lblk_journal_device = getattr(args, 'lblk_journal_device', False) if lblk_journal_device and not lblk: self.parser.error("--lblk-journal-device requires --lblk") + lblk_journal_only = getattr(args, 'lblk_journal_only', False) + if lblk_journal_only: + if not lblk: + self.parser.error("--lblk-journal-only requires --lblk") + if lblk_journal_device: + self.parser.error("--lblk-journal-only and --lblk-journal-device exclude each other") + if not (blk_names or blk_serials): + self.parser.error("--lblk-journal-only needs the journal disk or partition named with " + "--blk-names or --blk-serials") + lblk_array_only = getattr(args, 'lblk_array_only', False) + if lblk_array_only: + if not lblk: + self.parser.error("--lblk-array-only requires --lblk") + if lblk_journal_only or lblk_journal_device or blk_names or blk_names_exclude or blk_serials: + self.parser.error("--lblk-array-only selects no local device: not with --blk-names, " + "--blk-names-exclude, --blk-serials, --lblk-journal-only or " + "--lblk-journal-device") lblk_selection = None if use_lblk: if not lblk: @@ -282,6 +302,8 @@ def storage_node__configure(self, sub_command, args): "names_exclude": [str(x) for x in blk_names_exclude.split(',')] if blk_names_exclude else None, "serials": [str(x) for x in blk_serials.split(',')] if blk_serials else None, "journal_device": bool(lblk_journal_device), + "journal_only": bool(lblk_journal_only), + "array_only": bool(lblk_array_only), } # The core split is decided by the cluster's vcpu-count when the node # is added or restarted; here the default heuristic lays out a usable @@ -505,20 +527,6 @@ def storage_node__reset_device(self, sub_command, args): def storage_node__restart_device(self, sub_command, args): return device_controller.restart_device(args.device_id, args.force) - def storage_node__resize_device(self, sub_command, args): - try: - result = device_controller.resize_device(args.device_id) - except ValueError as e: - print(e) - return False - if not result["resized"]: - return f"Device size unchanged: {utils.humanbytes(result['new_size'])}" - out = (f"Device grown from {utils.humanbytes(result['old_size'])} " - f"to {utils.humanbytes(result['new_size'])}") - if result["warning"]: - out += f"\nWarning: {result['warning']}" - return out - def storage_node__add_device(self, sub_command, args): return device_controller.add_device(args.device_id) @@ -832,25 +840,93 @@ def cluster__change_name(self, sub_command, args): return True def cluster__grow_devices(self, sub_command, args): - result = device_controller.grow_cluster_devices( - args.cluster_id, rebalance=not getattr(args, 'no_rebalance', False)) - lines = [f"{r['device_id']}: {utils.humanbytes(r['old_size'])} -> {utils.humanbytes(r['new_size'])}" - for r in result["devices"] if r["resized"]] - lines += [f"{dev_id}: failed: {err}" for dev_id, err in result["errors"].items()] - rebalance = result.get("rebalance") - if rebalance and rebalance["started"]: - lines.append("The devices have grown unevenly: device weights set to the new sizes, " - "cluster rebalancing started.") - elif rebalance and rebalance["weights_updated"]: - lines.append(f"The devices have grown unevenly: device weights set to the new sizes. " - f"{rebalance['reason']}") - elif rebalance: - lines.append(f"Warning: the devices have grown unevenly. {rebalance['reason']}") - elif not result["uniform"]: - lines.append("Warning: the data devices of the cluster have grown unevenly; the added capacity is " - "only fully usable once all of them have grown by the same factor, or after " - "grow-devices without --no-rebalance has updated the weights.") - return "\n".join(lines) or "No device has grown" + lun_factor = getattr(args, 'lun_factor', None) + if lun_factor is not None: + try: + lun_factor = float(lun_factor) + except ValueError: + print(f"--lun-factor must be a number, not {lun_factor!r}") + return False + extra: dict = {"lun_factor": lun_factor} if lun_factor is not None else {} + lun_size = int(getattr(args, 'lun_size', 0) or 0) + if lun_size: + if lun_factor is not None: + print("Give --lun-factor or --lun-size, not both") + return False + extra = {"lun_size": lun_size} + try: + result = device_controller.grow_cluster_devices(args.cluster_id, **extra) + except ValueError as e: + print(e) + return False + lines = [f"LUN {r['lun_id']} of {dev_id}: {utils.humanbytes(r['old_size'])} -> " + f"{utils.humanbytes(r['new_size'])} at the array" + for dev_id, r in (result.get("luns") or {}).items()] + lines += [f"{dev_id}: {err}" for dev_id, err in result["errors"].items()] + if not result["grown"]: + lines.append(result["reason"]) + return "\n".join(lines) + lines.append(f"All data devices grown by factor {result['factor']:.4f}:") + for r in result["devices"]: + line = f"{r['device_id']}: {utils.humanbytes(r['old_size'])} -> {utils.humanbytes(r['new_size'])}" + if r["unused_size"]: + line += f" ({utils.humanbytes(r['unused_size'])} of the block device unused)" + lines.append(line) + return "\n".join(lines) + + def cluster__set_array(self, sub_command, args): + options = {} + if args.options: + try: + options = json.loads(args.options) + except json.JSONDecodeError as e: + raise ValueError(f"--options is not valid JSON: {e}") + if not isinstance(options, dict): + raise ValueError("--options must be a JSON object") + credentials = {} + if args.credentials_file: + try: + raw = json.loads(Path(args.credentials_file).read_text()) + except (OSError, json.JSONDecodeError) as e: + # Never the file content: it is the secret. + raise ValueError(f"--credentials-file could not be read as JSON: {type(e).__name__}") + if not isinstance(raw, dict) or not all(isinstance(v, str) for v in raw.values()): + raise ValueError("--credentials-file must hold a JSON object of strings") + # Wrapped at the boundary; unwrapped only by the KMS and the plugin. + credentials = {str(k): SecretStr(v) for k, v in raw.items()} + credential_ref = None + if getattr(args, "credential_ref", None): + try: + credential_ref = json.loads(args.credential_ref) + except json.JSONDecodeError as e: + raise ValueError(f"--credential-ref is not valid JSON: {e}") + if not isinstance(credential_ref, dict): + raise ValueError("--credential-ref must be a JSON object") + journal_options = None + if getattr(args, "journal_options", None): + try: + journal_options = json.loads(args.journal_options) + except json.JSONDecodeError as e: + raise ValueError(f"--journal-options is not valid JSON: {e}") + if not isinstance(journal_options, dict): + raise ValueError("--journal-options must be a JSON object") + view = array_controller.register_array( + args.cluster_id, args.plugin, args.endpoint or "", options, credentials, + int(args.lun_size or 0), credential_ref=credential_ref, + journal=getattr(args, "journal", None) or "local", + journal_lun_size=int(getattr(args, "journal_lun_size", 0) or 0), + journal_options=journal_options) + return _format_result([view], json=getattr(args, "json", False)) + + def cluster__get_array(self, sub_command, args): + arrays = array_controller.get_arrays(args.cluster_id) + if not arrays: + return "No array registered" + return _format_result(arrays, json=getattr(args, "json", False)) + + def cluster__remove_array(self, sub_command, args): + array_controller.delete_array(args.cluster_id, args.array_id, force=args.force) + return True def cluster__complete_expand(self, sub_command, args): cluster_ops.cluster_expand(args.cluster_id) @@ -1968,6 +2044,9 @@ def cluster_add(self, args): max_queue_size = args.max_queue_size inflight_io_threshold = args.inflight_io_threshold strict_node_anti_affinity = args.strict_node_anti_affinity + raid_devices = getattr(args, 'raid_devices', False) + multi_attach = getattr(args, 'multi_attach', False) + ftt = getattr(args, 'ftt', None) is_single_node = args.is_single_node client_data_nic = args.client_data_nic enable_failure_domain = getattr(args, 'enable_failure_domain', False) @@ -1999,6 +2078,10 @@ def cluster_add(self, args): sync_replication=getattr(args, 'sync_replication', False), page_copy=getattr(args, 'page_copy', False), md_journal=md_journal, + raid_devices=raid_devices, + multi_attach=multi_attach, + ftt=ftt, + multi_attach_warm_standby=getattr(args, 'multi_attach_warm_standby', False), container_image_prefix=args.container_image_prefix, ) @@ -2029,6 +2112,9 @@ def cluster_create(self, args): inflight_io_threshold = args.inflight_io_threshold disable_monitoring = args.disable_monitoring strict_node_anti_affinity = args.strict_node_anti_affinity + raid_devices = getattr(args, 'raid_devices', False) + multi_attach = getattr(args, 'multi_attach', False) + ftt = getattr(args, 'ftt', None) name = args.name tls_secret = args.tls_secret ingress_host_source = args.ingress_host_source @@ -2079,6 +2165,10 @@ def cluster_create(self, args): sync_replication=getattr(args, 'sync_replication', False), page_copy=getattr(args, 'page_copy', False), md_journal=md_journal, + raid_devices=raid_devices, + multi_attach=multi_attach, + ftt=ftt, + multi_attach_warm_standby=getattr(args, 'multi_attach_warm_standby', False), container_image_prefix=args.container_image_prefix, ) diff --git a/simplyblock_core/array/__init__.py b/simplyblock_core/array/__init__.py new file mode 100644 index 0000000000..85c97c2237 --- /dev/null +++ b/simplyblock_core/array/__init__.py @@ -0,0 +1,409 @@ +"""Storage-array integration of the multi-attach array LUN mode. + +A multi-attach cluster (``Cluster.multi_attach``) runs on array LUNs that are +presented to several storage nodes. Before a LUN is reopened on another node +the control plane must fence the node that served it at the array: that node's +host multipath layer may still hold queued writes, and once released they would +land after, and overwrite, newer data written through the new node. + +This package holds the plugin interface (:class:`ArrayPlugin`: LUN create / +attach / detach / resize / get / delete / list and fence / unfence) and a +registry that loads a vendor plugin by name from the entry-point group +``simplyblock.array_plugins``. The array's credentials live in the KMS at +``cluster//array/``; the Cluster record keeps the plugin +name, endpoint and options only. Built-in plugins: ``aws-ebs`` +(:mod:`.aws_ebs`) and the SAN arrays (:mod:`.san`). Registration and the LUN lifecycle are in +:mod:`simplyblock_core.controllers.array_controller`. + +Plugins are found only through the entry-point group or an explicit +:func:`register_plugin` call. The test double in :mod:`.fake` is never +registered by default, so a cluster cannot be configured with a plugin that +fences nothing. +""" + +from __future__ import annotations + +import abc +import json +import logging +import threading +import time +from importlib import metadata + +logger = logging.getLogger() + +#: The entry-point group vendor plugins register under; the entry point's +#: name is the value of ``Cluster.array_plugin``. +ENTRY_POINT_GROUP = "simplyblock.array_plugins" + +#: Plugins shipped with simplyblock (also declared as entry points in +#: pyproject.toml). +BUILTIN_PLUGINS = { + "aws-ebs": "simplyblock_core.array.aws_ebs:AwsEbsPlugin", + # SAN arrays: the storage_arrays adapters (simplyblock_core.array.san). + "pure-flasharray": "simplyblock_core.array.san:PureFlashArrayPlugin", + "netapp-ontap": "simplyblock_core.array.san:NetAppOntapPlugin", + "dell-powerstore": "simplyblock_core.array.san:DellPowerStorePlugin", + "dell-powermax": "simplyblock_core.array.san:DellPowerMaxPlugin", + "dell-powerflex": "simplyblock_core.array.san:DellPowerFlexPlugin", + "hpe-wsapi": "simplyblock_core.array.san:HpeWsapiPlugin", + "hpe-nimble": "simplyblock_core.array.san:HpeNimblePlugin", + "ibm-flashsystem": "simplyblock_core.array.san:IbmFlashSystemPlugin", + "hitachi-vsp": "simplyblock_core.array.san:HitachiVspPlugin", + "huawei-dorado": "simplyblock_core.array.san:HuaweiDoradoPlugin", + "infinidat-infinibox": "simplyblock_core.array.san:InfinidatInfiniBoxPlugin", +} + + +class ArrayError(Exception): + """An array operation failed, or no plugin could be loaded.""" + + +class ArrayNotConfigured(ArrayError): + """The cluster names no array plugin.""" + + +#: Tag every LUN a plugin creates carries, with the cluster id as its value. +#: A plugin acts only on LUNs carrying its cluster's tag (ForeignLunError). +TAG_CLUSTER_ID = "simplyblock-cluster-id" +#: Tag carrying the LUN name the control plane asked for. +TAG_LUN_NAME = "simplyblock-lun-name" + + +class ForeignLunError(ArrayError): + """The LUN was not created by this cluster's plugin: it carries no, or + another, cluster tag. It is never touched.""" + + +class ArrayPlugin(abc.ABC): + """The array operations the control plane needs. + + simplyblock creates, presents (multi-attaches), grows, fences and deletes + the LUNs of a multi-attach cluster itself, through one plugin per vendor. + + Identities: ``lun_id`` is the array's own id of a LUN, as returned by + :meth:`create_lun` (``NVMeDevice.array_lun_id``); a plugin should also + accept the LUN's ``wwn`` as the hosts report it where it can map it + (phase 3 fenced by ``NVMeDevice.lun_wwn``). ``host_id`` is the array's id + of a storage node's host, as returned by :meth:`ensure_host` + (``StorageNode.array_host_id``). + + Every operation is idempotent: creating a LUN that exists returns it, + attaching an attached LUN, detaching a detached one and fencing a fenced + host succeed. A method reports failure by raising (preferably + :class:`ArrayError`); ``fence`` / ``unfence`` may also return a falsy + value. A plugin acts only on LUNs it created for ``cluster_id``: it tags + each with :data:`TAG_CLUSTER_ID` and raises :class:`ForeignLunError` for + any other LUN. + + Credentials (``connect``) are a flat dict of ``SecretStr`` (from the + KMS); a plugin unwraps them only where it hands them to its SDK and never + logs them. + """ + + def __init__(self, endpoint: str = "", options: dict | None = None, credentials=None, + cluster_id: str = ""): + self.endpoint = endpoint + self.options = dict(options or {}) + self.credentials = dict(credentials or {}) + self.cluster_id = cluster_id + + def __repr__(self): + # Never the credentials. + return f"{type(self).__name__}(endpoint={self.endpoint!r}, cluster_id={self.cluster_id!r})" + + def connect(self, endpoint: str, credentials: dict, options: dict) -> None: + """Set up the session to the array. Called once per instance, before + any other operation; raises when the array cannot be reached or the + credentials are refused. The default only stores the arguments.""" + self.endpoint = endpoint or "" + self.credentials = dict(credentials or {}) + self.options = dict(options or {}) + + def capabilities(self) -> dict: + """``{"protocols": [...], "supports_reservations": bool, + "max_attachments": int}`` (0 = no limit known).""" + return {"protocols": [], "supports_reservations": False, "max_attachments": 0} + + # ------------------------------------------------------------ ownership + + def cluster_tags(self, tags: dict | None = None, name: str = "") -> dict: + """``tags`` plus the ownership tags every created LUN carries.""" + if not self.cluster_id: + raise ArrayError("the plugin has no cluster id; it cannot tag what it creates") + out = {str(k): str(v) for k, v in (tags or {}).items()} + out[TAG_CLUSTER_ID] = self.cluster_id + if name: + out[TAG_LUN_NAME] = name + return out + + def check_owned(self, lun_id: str, tags: dict | None) -> None: + """Raise :class:`ForeignLunError` unless ``tags`` (the LUN's tags on the + array) carry this cluster's id.""" + if not self.cluster_id or (tags or {}).get(TAG_CLUSTER_ID) != self.cluster_id: + raise ForeignLunError( + f"LUN {lun_id} was not created by simplyblock cluster {self.cluster_id or '?'}; " + "it is not touched") + + # ---------------------------------------------------------- operations + + @abc.abstractmethod + def ensure_host(self, name: str, initiators: dict) -> str: + """The array's id of the host ``name`` with ``initiators`` (vendor + specific: NQNs, WWPNs, IQNs, an EC2 instance id), created when the + array keeps host objects.""" + + @abc.abstractmethod + def create_lun(self, name: str, size_bytes: int, tags: dict) -> dict: + """Create LUN ``name`` (at least ``size_bytes``) tagged with ``tags`` + plus :meth:`cluster_tags`, or return the existing one of that name. + Returns ``{"id", "wwn", "size"}``; ``wwn`` is the identity the hosts + report for it (lsblk WWN or SERIAL).""" + + @abc.abstractmethod + def attach(self, lun_id: str, host_id: str) -> None: + """Present the LUN to the host (multi-attach: in addition to the + hosts it is presented to already).""" + + @abc.abstractmethod + def detach(self, lun_id: str, host_id: str, force: bool = False) -> None: + """Stop presenting the LUN to the host. ``force``: also when the host + does not cooperate; returns only once the host can no longer reach + the LUN.""" + + @abc.abstractmethod + def resize(self, lun_id: str, size_bytes: int) -> None: + """Grow the LUN to at least ``size_bytes``; returns once the hosts can + see the new size (a rescan away). Shrinking is refused.""" + + @abc.abstractmethod + def get(self, lun_id: str) -> dict: + """``{"id", "wwn", "size", "hosts": [host ids it is presented to]}``.""" + + @abc.abstractmethod + def delete_lun(self, lun_id: str) -> None: + """Delete the LUN, detaching it from every host first. Deleting a LUN + that is gone succeeds.""" + + @abc.abstractmethod + def list_luns(self) -> list[dict]: + """Every LUN of this cluster (tagged with its id), as :meth:`get` + returns them.""" + + # ------------------------------------------------------------- fencing + + def fence(self, lun_id: str, host_id: str) -> bool: + """Revoke the host's access to the LUN so no I/O of it, including + I/O queued in its multipath layer, can reach the LUN any more. + Returns only once the array has applied it. Default: a forced + detach.""" + self.detach(lun_id, host_id, force=True) + return True + + def unfence(self, lun_id: str, host_id: str) -> bool: + """Give the host access to the LUN again. Default: attach.""" + self.attach(lun_id, host_id) + return True + + +_registry: dict[str, type[ArrayPlugin]] = {} +_registry_lock = threading.Lock() + + +def register_plugin(name: str, cls: type[ArrayPlugin]) -> None: + """Register ``cls`` under ``name`` in this process (tests; built-in + plugins). An entry point of the same name is shadowed.""" + if not (isinstance(cls, type) and issubclass(cls, ArrayPlugin)): + raise TypeError(f"{cls!r} is not an ArrayPlugin") + with _registry_lock: + _registry[name] = cls + + +def unregister_plugin(name: str) -> None: + with _registry_lock: + _registry.pop(name, None) + + +def _entry_point(name: str): + found = list(metadata.entry_points(group=ENTRY_POINT_GROUP, name=name)) + return found[0] if found else None + + +def plugin_class(name: str) -> type[ArrayPlugin]: + """The plugin class registered as ``name``. + + Raises: + ArrayError: no plugin of that name, or it is not an ArrayPlugin. + """ + with _registry_lock: + cls = _registry.get(name) + if cls is not None: + return cls + ep = _entry_point(name) + if ep is None and name in BUILTIN_PLUGINS: + # The package's own entry point, when it runs from a source tree that + # was never installed (entry points come from installed metadata). + ep = metadata.EntryPoint(name=name, value=BUILTIN_PLUGINS[name], group=ENTRY_POINT_GROUP) + if ep is None: + raise ArrayError(f"no array plugin {name!r} in entry-point group {ENTRY_POINT_GROUP}") + try: + cls = ep.load() + except Exception as e: + raise ArrayError(f"array plugin {name!r} failed to load: {e}") from e + if not (isinstance(cls, type) and issubclass(cls, ArrayPlugin)): + raise ArrayError(f"array plugin {name!r} ({ep.value}) is not an ArrayPlugin") + return cls + + +def load_plugin(name: str, endpoint: str = "", options: dict | None = None, + credentials=None, cluster_id: str = "") -> ArrayPlugin: + """A connected instance of plugin ``name`` for ``endpoint``. + + Raises: + ArrayNotConfigured: ``name`` is empty. + ArrayError: the plugin cannot be found, loaded, constructed or + connected. + """ + if not name: + raise ArrayNotConfigured("no array plugin is configured") + cls = plugin_class(name) + try: + plugin = cls(endpoint=endpoint, options=options, credentials=credentials, + cluster_id=cluster_id) + except Exception as e: + raise ArrayError(f"array plugin {name!r} could not be set up: {e}") from e + try: + plugin.connect(endpoint, dict(credentials or {}), dict(options or {})) + except Exception as e: + # An SDK error message does not carry the credentials, and they are + # never formatted into one here. + raise ArrayError(f"array plugin {name!r} could not connect to {endpoint or 'the array'}: {e}") from e + return plugin + + +def credential_context(array_id: str, endpoint: str) -> str: + """What an array's credentials are bound to: the array id and the + scheme/host/port of its endpoint. A credential wrapped for one array is + refused for another endpoint, so rewriting the endpoint in the database + cannot send it elsewhere.""" + from urllib.parse import urlsplit + ep = (endpoint or "").strip() + parts = urlsplit(ep) + if parts.scheme and parts.hostname: + port = parts.port or (443 if parts.scheme == "https" else 80) + ep = f"{parts.scheme}://{parts.hostname.lower()}:{port}" + return f"array/{array_id}/{ep}" + + +def credential_ref_view(cluster) -> dict: + """The cluster's array credential reference without anything secret: + the kind, and for an external secret where it lives.""" + ref = dict(getattr(cluster, "array_credential_ref", None) or {}) + if not ref: + return {} + if ref.get("kind") == "managed": + return {"kind": "managed", "backend": ref.get("backend", "")} + return {k: v for k, v in ref.items() if k in ("kind", "mount", "path", "version", "uid", "field")} + + +def array_credentials(cluster) -> dict: + """The credentials of the cluster's array (``SecretStr`` values), or + ``{}`` when the array was registered without any (e.g. the EBS plugin on + the instance role) or no array is registered. + + Read through ``Cluster.array_credential_ref`` (wrapped by the cluster's + KMS, or an external Vault KV / KMIP secret); arrays registered before + references existed are read from their KMS path. + + Raises: + ArrayError: a secret is recorded but cannot be read. + """ + array_id = getattr(cluster, "array_id", "") or "" + if not array_id or getattr(cluster, "array_has_credentials", False) is not True: + return {} + ref = getattr(cluster, "array_credential_ref", None) or {} + if ref: + from simplyblock_core import credentials as creds_mod + try: + resolved = creds_mod.CredentialProvider(cluster, ttl_s=0).resolve( + ref, credential_context(array_id, getattr(cluster, "array_endpoint", "") or "")) + except creds_mod.CredentialError as e: + raise ArrayError(f"array credentials could not be read: {e}") from e + return {name: resolved.get(name) for name in resolved.field_names + if name != creds_mod.CONTEXT_FIELD} + from simplyblock_core import kms + try: + with kms.create_kms_connection(cluster) as conn: + return conn.get_secret(kms.array_secret_path(cluster.get_id(), array_id)) + except Exception as e: + raise ArrayError(f"array credentials could not be read from the KMS: {e}") from e + + +def plugin_for_cluster(cluster, extra_options: dict | None = None) -> ArrayPlugin: + """The connected array plugin a cluster is configured with + (``Cluster.array_plugin`` / ``array_endpoint`` / ``array_options``, + credentials from the KMS). ``extra_options`` update the options for this + instance (the journal LUNs' ``Cluster.array_journal_options``). + + Raises: + ArrayNotConfigured: the cluster names no plugin. + ArrayError: it cannot be loaded or connected. + """ + name = getattr(cluster, "array_plugin", "") or "" + if not name: + raise ArrayNotConfigured("no array plugin is configured") + return load_plugin( + name, + getattr(cluster, "array_endpoint", "") or "", + dict(getattr(cluster, "array_options", None) or {}, **(extra_options or {})), + credentials=array_credentials(cluster), + cluster_id=cluster.get_id(), + ) + + +# cluster id -> (configuration key, monotonic time connected, plugin) +_connected: dict[str, tuple] = {} +_connected_lock = threading.Lock() + + +def _config_key(cluster) -> tuple: + return (getattr(cluster, "array_plugin", "") or "", + getattr(cluster, "array_endpoint", "") or "", + json.dumps(getattr(cluster, "array_options", None) or {}, sort_keys=True, default=str), + getattr(cluster, "array_id", "") or "", + getattr(cluster, "array_has_credentials", False) is True, + json.dumps(getattr(cluster, "array_credential_ref", None) or {}, sort_keys=True)) + + +def cached_plugin_for_cluster(cluster, max_age: float | None = None) -> ArrayPlugin: + """:func:`plugin_for_cluster`, connected once and reused for + ``constants.ARRAY_PLUGIN_CACHE_SEC`` while the cluster's array + configuration is unchanged. For the LUN takeover's fence, which every + volume of a 1+0 cluster waits on: connecting validates the credentials + first (about a second on EBS). A caller whose call fails on the cached + instance drops it with :func:`forget_plugin` and connects again. + ``max_age`` (seconds) connects anew when the cached one is older (a + refresh ahead of the expiry, so a fence never waits for it).""" + from simplyblock_core import constants + cid = cluster.get_id() + key = _config_key(cluster) + # The class too: a plugin registered anew under the same name (tests). + key = (plugin_class(key[0]) if key[0] else None,) + key + with _connected_lock: + hit = _connected.get(cid) + limit = constants.ARRAY_PLUGIN_CACHE_SEC if max_age is None else max_age + if hit and hit[0] == key and time.monotonic() - hit[1] < limit: + return hit[2] + plugin = plugin_for_cluster(cluster) + with _connected_lock: + _connected[cid] = (key, time.monotonic(), plugin) + return plugin + + +def forget_plugin(cluster_id: str | None = None) -> None: + """Drop the cached connection of one cluster (all with None).""" + with _connected_lock: + if cluster_id is None: + _connected.clear() + else: + _connected.pop(cluster_id, None) diff --git a/simplyblock_core/array/aws_ebs.py b/simplyblock_core/array/aws_ebs.py new file mode 100644 index 0000000000..70d42491cb --- /dev/null +++ b/simplyblock_core/array/aws_ebs.py @@ -0,0 +1,437 @@ +"""AWS EBS io2 Multi-Attach as the "array" of a multi-attach cluster. + +Each storage node's LUN is an io2 (or io1) volume created with +``MultiAttachEnabled``, attached to the owner and to its LVS secondary and +tertiary. All instances must be Nitro instances in the one availability zone +of the cluster (``options["az"]``); a multi-attach volume cannot leave its AZ. + +Identities: + +* LUN id: the volume id (``vol-0123...``). The host-side identity (``wwn``) + is the NVMe serial EBS gives the volume's namespace, ``vol0123...`` (the id + without its dash), which lsblk reports as SERIAL on every instance the + volume is attached to; ``NVMeDevice.lun_wwn`` and the agent's + ``find_lun_devices`` match on it. Both forms are accepted wherever a LUN id + is expected. +* host id: the EC2 instance id (``ensure_host`` takes + ``{"ec2_instance_id": ...}``; the node agent reports it from the instance + metadata service as ``cloud_instance.id``, StorageNode.cloud_instance_id). + +Ownership: every volume created here is tagged ``simplyblock-cluster-id = +``; a volume without this cluster's tag is never attached, +detached, resized or deleted (:class:`ForeignLunError`). + +Fencing: ``fence`` is a forced ``DetachVolume`` of the failed instance's +attachment, and returns only once EC2 reports the attachment gone. A +detached EBS volume is hot-removed from the instance's PCIe bus; the instance +can issue no further I/O to it, queued or new. + +Credentials (KMS secret, all optional): ``aws_access_key_id``, +``aws_secret_access_key``, ``aws_session_token``. Without them the default +boto3 chain applies (the control plane's instance role / IRSA). + +Options: ``az`` (required), ``region`` (default: ``az`` without its letter), +``volume_type`` (``io2`` default, or ``io1``), ``iops`` (default 3000), +``encrypted`` (default true), ``kms_key_id``, ``tags`` (dict, added to every +volume), ``poll_interval`` (s, default 2), ``fence_poll_interval`` (s, default +0.2: a fence holds the volumes), ``timeout`` (s, default 300). +``endpoint`` (the registration's endpoint) is an EC2 endpoint URL override +(VPC endpoint), empty for the regional default. +""" + +from __future__ import annotations + +import logging +import math +import time + +from pydantic import SecretStr + +from simplyblock_core.array import TAG_CLUSTER_ID, TAG_LUN_NAME, ArrayError, ArrayPlugin + +logger = logging.getLogger() + +GIB = 1024 ** 3 +#: Smallest io1/io2 volume. +MIN_SIZE_GIB = 4 +#: EBS Multi-Attach: at most 16 Nitro instances per volume. +MAX_ATTACHMENTS = 16 +MULTI_ATTACH_TYPES = ("io2", "io1") +_LIVE_ATTACHMENT = ("attaching", "attached", "busy") +_GONE_VOLUME = ("deleting", "deleted", "error") + + +def volume_wwn(volume_id: str) -> str: + """The NVMe serial of an EBS volume as an instance sees it.""" + return volume_id.replace("-", "", 1) + + +def volume_id_of(lun: str) -> str: + """``vol-0abc`` from either the volume id or its NVMe serial.""" + lun = (lun or "").strip() + if lun.startswith("vol-"): + return lun + if lun.startswith("vol") and len(lun) > 3: + return "vol-" + lun[3:] + raise ArrayError(f"{lun!r} is no EBS volume id") + + +def _plain(value) -> str: + if isinstance(value, SecretStr): + return value.get_secret_value() + return str(value or "") + + +class AwsEbsPlugin(ArrayPlugin): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.ec2 = None + self._sleep = time.sleep + + # ------------------------------------------------------------- session + + def connect(self, endpoint, credentials, options): + super().connect(endpoint, credentials, options) + az = self.options.get("az") or "" + if not az: + raise ArrayError("aws-ebs: option 'az' (the availability zone of every node) is required") + vtype = self.options.get("volume_type", "io2") + if vtype not in MULTI_ATTACH_TYPES: + raise ArrayError(f"aws-ebs: volume_type must be one of {MULTI_ATTACH_TYPES} (Multi-Attach)") + try: + import boto3 + except ImportError as e: # pragma: no cover - boto3 is a dependency + raise ArrayError("aws-ebs: boto3 is not installed") from e + creds = self.credentials + key_id = _plain(creds.get("aws_access_key_id")) + secret = _plain(creds.get("aws_secret_access_key")) + if bool(key_id) != bool(secret): + raise ArrayError("aws-ebs: aws_access_key_id and aws_secret_access_key go together") + session = boto3.session.Session( + aws_access_key_id=key_id or None, + aws_secret_access_key=secret or None, + aws_session_token=_plain(creds.get("aws_session_token")) or None, + region_name=self.region, + ) + self.ec2 = session.client("ec2", endpoint_url=endpoint or None) + # Validates the credentials and the zone in one call. + try: + zones = self.ec2.describe_availability_zones(ZoneNames=[az])["AvailabilityZones"] + except Exception as e: + raise ArrayError(f"aws-ebs: availability zone {az} not usable: {self._err(e)}") from e + if not zones: + raise ArrayError(f"aws-ebs: availability zone {az} not found") + + @property + def region(self) -> str: + return self.options.get("region") or (self.options.get("az") or "")[:-1] + + @property + def _poll(self) -> float: + return float(self.options.get("poll_interval", 2)) + + @property + def _fence_poll(self) -> float: + # A fence holds every volume of a 1+0 cluster: poll its detach fast. + return float(self.options.get("fence_poll_interval", 0.2)) + + @property + def _timeout(self) -> float: + return float(self.options.get("timeout", 300)) + + @staticmethod + def _err(e) -> str: + resp = getattr(e, "response", None) or {} + err = resp.get("Error") or {} + if err: + return f"{err.get('Code')}: {err.get('Message')}" + return str(e) + + @staticmethod + def _code(e) -> str: + return ((getattr(e, "response", None) or {}).get("Error") or {}).get("Code", "") + + def _client(self): + if self.ec2 is None: + raise ArrayError("aws-ebs: not connected") + return self.ec2 + + def capabilities(self): + return {"protocols": ["nvme"], "supports_reservations": True, + "max_attachments": MAX_ATTACHMENTS} + + # ------------------------------------------------------------- helpers + + def _describe(self, volume_id): + try: + vols = self._client().describe_volumes(VolumeIds=[volume_id])["Volumes"] + except Exception as e: + if self._code(e) == "InvalidVolume.NotFound": + return None + raise ArrayError(f"aws-ebs: describe {volume_id}: {self._err(e)}") from e + return vols[0] if vols else None + + @staticmethod + def _tags(vol) -> dict: + return {t["Key"]: t["Value"] for t in vol.get("Tags") or []} + + def _owned(self, lun): + vid = volume_id_of(lun) + vol = self._describe(vid) + if vol is None or vol.get("State") in _GONE_VOLUME: + raise ArrayError(f"aws-ebs: volume {vid} not found") + self.check_owned(vid, self._tags(vol)) + return vol + + @staticmethod + def _attachment(vol, instance_id): + for att in vol.get("Attachments") or []: + if att.get("InstanceId") == instance_id: + return att + return None + + @staticmethod + def _summary(vol) -> dict: + return { + "id": vol["VolumeId"], + "wwn": volume_wwn(vol["VolumeId"]), + "size": int(vol.get("Size") or 0) * GIB, + "hosts": [a["InstanceId"] for a in vol.get("Attachments") or [] + if a.get("State") in _LIVE_ATTACHMENT], + "state": vol.get("State", ""), + } + + def _wait(self, what, check, poll=None): + """Poll ``check()`` until it returns a truthy value or the timeout.""" + deadline = time.monotonic() + self._timeout + while True: + result = check() + if result: + return result + if time.monotonic() >= deadline: + raise ArrayError(f"aws-ebs: timed out waiting for {what}") + self._sleep(self._poll if poll is None else poll) + + def _free_device_name(self, instance_id) -> str: + try: + res = self._client().describe_instances(InstanceIds=[instance_id]) + inst = res["Reservations"][0]["Instances"][0] + except Exception as e: + raise ArrayError(f"aws-ebs: describe instance {instance_id}: {self._err(e)}") from e + used = {m.get("DeviceName") for m in inst.get("BlockDeviceMappings") or []} + used.add(inst.get("RootDeviceName")) + # The device name is ignored for NVMe on Nitro but must be unique + # per instance. /dev/sd[f-z], then /dev/sd[f-z][a-z]. + letters = "fghijklmnopqrstuvwxyz" + candidates = [f"/dev/sd{a}" for a in letters] + candidates += [f"/dev/sd{a}{b}" for a in letters for b in "abcdefghijklmnopqrstuvwxyz"] + for name in candidates: + if name not in used: + return name + raise ArrayError(f"aws-ebs: no free device name on {instance_id}") + + # ---------------------------------------------------------- operations + + def ensure_host(self, name, initiators): + instance_id = (initiators or {}).get("ec2_instance_id") or "" + if not instance_id.startswith("i-"): + # The node agent reports the instance id only when its cloud + # probe runs (docker deployments start it with + # WITHOUT_CLOUD_INFO). The node's private IP identifies the + # instance as well, within the zone of the volumes. + instance_id = self._instance_by_private_ip(name, (initiators or {}).get("ip") or "") + try: + res = self._client().describe_instances(InstanceIds=[instance_id]) + inst = res["Reservations"][0]["Instances"][0] + except ArrayError: + raise + except Exception as e: + raise ArrayError(f"aws-ebs: instance {instance_id} of host {name}: {self._err(e)}") from e + az = (inst.get("Placement") or {}).get("AvailabilityZone", "") + if az != self.options.get("az"): + raise ArrayError( + f"aws-ebs: instance {instance_id} of host {name} is in {az}, the volumes in " + f"{self.options.get('az')}; Multi-Attach volumes cannot cross zones") + return instance_id + + def _instance_by_private_ip(self, name, ip): + if not ip: + raise ArrayError(f"aws-ebs: host {name} has no EC2 instance id and no IP") + try: + res = self._client().describe_instances(Filters=[ + {"Name": "private-ip-address", "Values": [ip]}, + {"Name": "availability-zone", "Values": [self.options.get("az") or ""]}, + {"Name": "instance-state-name", "Values": ["pending", "running"]}]) + except Exception as e: + raise ArrayError(f"aws-ebs: find the instance of host {name} ({ip}): {self._err(e)}") from e + ids = [i["InstanceId"] for r in res.get("Reservations") or [] for i in r.get("Instances") or []] + if len(ids) != 1: + raise ArrayError( + f"aws-ebs: host {name} has no EC2 instance id, and {len(ids)} running instances " + f"in {self.options.get('az')} have the private IP {ip}") + return ids[0] + + def create_lun(self, name, size_bytes, tags): + all_tags = self.cluster_tags(dict(self.options.get("tags") or {}, **(tags or {})), name) + all_tags.setdefault("Name", name) + try: + found = self._client().describe_volumes(Filters=[ + {"Name": f"tag:{TAG_CLUSTER_ID}", "Values": [self.cluster_id]}, + {"Name": f"tag:{TAG_LUN_NAME}", "Values": [name]}, + ])["Volumes"] + except Exception as e: + raise ArrayError(f"aws-ebs: look up volume {name}: {self._err(e)}") from e + live = [v for v in found if v.get("State") not in _GONE_VOLUME] + if len(live) > 1: + raise ArrayError(f"aws-ebs: {len(live)} volumes are named {name} in this cluster") + if live: + vol = live[0] + logger.info("aws-ebs: volume %s for %s exists", vol["VolumeId"], name) + else: + size_gib = max(MIN_SIZE_GIB, math.ceil(int(size_bytes) / GIB)) + kwargs = { + "AvailabilityZone": self.options["az"], + "Size": size_gib, + "VolumeType": self.options.get("volume_type", "io2"), + "Iops": int(self.options.get("iops", 3000)), + "MultiAttachEnabled": True, + "Encrypted": bool(self.options.get("encrypted", True)), + "TagSpecifications": [{ + "ResourceType": "volume", + "Tags": [{"Key": k, "Value": v} for k, v in sorted(all_tags.items())], + }], + } + if self.options.get("kms_key_id"): + kwargs["KmsKeyId"] = self.options["kms_key_id"] + try: + vol = self._client().create_volume(**kwargs) + except Exception as e: + raise ArrayError(f"aws-ebs: create volume {name}: {self._err(e)}") from e + logger.info("aws-ebs: created volume %s (%s GiB) for %s", vol["VolumeId"], size_gib, name) + vid = vol["VolumeId"] + + def _available(): + v = self._describe(vid) + if v is not None and v.get("State") == "error": + raise ArrayError(f"aws-ebs: volume {vid} failed to create") + return v if v is not None and v.get("State") in ("available", "in-use") else None + vol = self._wait(f"volume {vid} to become available", _available) + return {k: v for k, v in self._summary(vol).items() if k in ("id", "wwn", "size")} + + def attach(self, lun_id, host_id): + vol = self._owned(lun_id) + vid = vol["VolumeId"] + att = self._attachment(vol, host_id) + if att is None or att.get("State") not in _LIVE_ATTACHMENT: + live = [a for a in vol.get("Attachments") or [] if a.get("State") in _LIVE_ATTACHMENT] + if len(live) >= MAX_ATTACHMENTS: + raise ArrayError(f"aws-ebs: volume {vid} is attached to {MAX_ATTACHMENTS} instances") + device = self._free_device_name(host_id) + try: + self._client().attach_volume(VolumeId=vid, InstanceId=host_id, Device=device) + except Exception as e: + raise ArrayError(f"aws-ebs: attach {vid} to {host_id}: {self._err(e)}") from e + logger.info("aws-ebs: attaching %s to %s as %s", vid, host_id, device) + + def _attached(): + a = self._attachment(self._describe(vid) or {}, host_id) + return a is not None and a.get("State") == "attached" + self._wait(f"volume {vid} to attach to {host_id}", _attached) + + def fence(self, lun_id, host_id): + """A forced detach, polled every ``fence_poll_interval`` (0.2 s).""" + self.detach(lun_id, host_id, force=True, poll=self._fence_poll) + return True + + def detach(self, lun_id, host_id, force=False, poll=None): + vol = self._owned(lun_id) + vid = vol["VolumeId"] + att = self._attachment(vol, host_id) + if att is None or att.get("State") == "detached": + return + if att.get("State") != "detaching" or force: + try: + self._client().detach_volume(VolumeId=vid, InstanceId=host_id, Force=bool(force)) + except Exception as e: + if self._code(e) not in ("IncorrectState", "InvalidAttachment.NotFound"): + raise ArrayError(f"aws-ebs: detach {vid} from {host_id}: {self._err(e)}") from e + logger.info("aws-ebs: detaching %s from %s%s", vid, host_id, " (forced)" if force else "") + + def _detached(): + v = self._describe(vid) + a = self._attachment(v or {}, host_id) + return a is None or a.get("State") == "detached" + self._wait(f"volume {vid} to detach from {host_id}", _detached, poll) + + def resize(self, lun_id, size_bytes): + vol = self._owned(lun_id) + vid = vol["VolumeId"] + target = math.ceil(int(size_bytes) / GIB) + current = int(vol.get("Size") or 0) + if target < current: + raise ArrayError(f"aws-ebs: volume {vid} is {current} GiB; shrinking to {target} GiB is refused") + if target > current: + try: + self._client().modify_volume(VolumeId=vid, Size=target) + except Exception as e: + raise ArrayError(f"aws-ebs: grow {vid} to {target} GiB: {self._err(e)}") from e + logger.info("aws-ebs: growing %s from %s to %s GiB", vid, current, target) + + def _grown(): + try: + mods = self._client().describe_volumes_modifications( + VolumeIds=[vid])["VolumesModifications"] + except Exception as e: + if self._code(e) == "InvalidVolumeModification.NotFound": + mods = [] + else: + raise ArrayError(f"aws-ebs: modification of {vid}: {self._err(e)}") from e + for m in mods: + state = m.get("ModificationState") + if state == "failed": + raise ArrayError(f"aws-ebs: growing {vid} failed: {m.get('StatusMessage', '')}") + if state == "modifying": + return False + # "optimizing" or "completed": the new size is usable. + return True + self._wait(f"volume {vid} to grow", _grown) + + def get(self, lun_id): + return self._summary(self._owned(lun_id)) + + def delete_lun(self, lun_id): + vid = volume_id_of(lun_id) + vol = self._describe(vid) + if vol is None or vol.get("State") in ("deleting", "deleted"): + return + self.check_owned(vid, self._tags(vol)) + for att in vol.get("Attachments") or []: + if att.get("State") in _LIVE_ATTACHMENT or att.get("State") == "detaching": + self.detach(vid, att["InstanceId"], force=True) + + def _available(): + v = self._describe(vid) + return v is None or v.get("State") == "available" + self._wait(f"volume {vid} to become available", _available) + try: + self._client().delete_volume(VolumeId=vid) + except Exception as e: + if self._code(e) != "InvalidVolume.NotFound": + raise ArrayError(f"aws-ebs: delete {vid}: {self._err(e)}") from e + logger.info("aws-ebs: deleted volume %s", vid) + + def list_luns(self): + if not self.cluster_id: + raise ArrayError("aws-ebs: no cluster id to list the volumes of") + out = [] + try: + pages = self._client().get_paginator("describe_volumes").paginate( + Filters=[{"Name": f"tag:{TAG_CLUSTER_ID}", "Values": [self.cluster_id]}]) + for page in pages: + for vol in page["Volumes"]: + if vol.get("State") not in _GONE_VOLUME: + out.append(self._summary(vol)) + except Exception as e: + raise ArrayError(f"aws-ebs: list volumes: {self._err(e)}") from e + return out diff --git a/simplyblock_core/array/fake.py b/simplyblock_core/array/fake.py new file mode 100644 index 0000000000..4986e25004 --- /dev/null +++ b/simplyblock_core/array/fake.py @@ -0,0 +1,178 @@ +"""In-memory array plugin for tests. Never registered by default.""" + +from __future__ import annotations + +import threading +from typing import ClassVar + +from simplyblock_core.array import ArrayError, ArrayPlugin, ForeignLunError + + +class FakeArrayPlugin(ArrayPlugin): + """An array kept in memory. + + ``calls`` records every operation as ``(op, *args)``; ``fence`` / + ``unfence`` as ``(op, lun, host)`` and the fenced pairs in ``fenced``. + ``luns``: lun id -> ``{"id", "name", "wwn", "size", "tags", "hosts"}``; + ``hosts``: host id -> ``{"name", "initiators"}``. ``fail`` = operation + names that raise :class:`ArrayError`; ``fail_falsy`` the ones that return + False instead (fence / unfence only). ``connected`` holds the + ``(endpoint, credentials, options)`` of every ``connect`` (not in + ``calls``). + + State is per class (``reset()`` clears it), because the control plane + builds a fresh instance from the cluster record each time.""" + + calls: ClassVar[list] = [] + fenced: ClassVar[set] = set() + fail: ClassVar[set] = set() + fail_falsy: ClassVar[set] = set() + luns: ClassVar[dict] = {} + hosts: ClassVar[dict] = {} + connected: ClassVar[list] = [] + max_attachments: ClassVar[int] = 16 + _lock = threading.RLock() + _next: ClassVar[int] = 0 + + @classmethod + def reset(cls): + with cls._lock: + cls.calls = [] + cls.fenced = set() + cls.fail = set() + cls.fail_falsy = set() + cls.luns = {} + cls.hosts = {} + cls.connected = [] + cls.max_attachments = 16 + cls._next = 0 + + def _record(self, op, *args): + with self._lock: + type(self).calls.append((op, *args)) + if op in self.fail: + raise ArrayError(f"fake array: {op} {args} failed") + + def _lun(self, lun_id): + lun = self.luns.get(lun_id) + if lun is None: + # Accept the WWN as well, as the EBS plugin does. + lun = next((x for x in self.luns.values() if x["wwn"] == lun_id), None) + if lun is None: + raise ArrayError(f"fake array: no LUN {lun_id}") + self.check_owned(lun_id, lun["tags"]) + return lun + + def connect(self, endpoint, credentials, options): + super().connect(endpoint, credentials, options) + if "connect" in self.fail: + raise ArrayError("fake array: connect refused") + with self._lock: + type(self).connected.append((endpoint, dict(credentials or {}), dict(options or {}))) + + def capabilities(self): + return {"protocols": ["fake"], "supports_reservations": False, + "max_attachments": self.max_attachments} + + def ensure_host(self, name, initiators): + self._record("ensure_host", name) + host_id = f"host-{name}" + with self._lock: + type(self).hosts[host_id] = {"name": name, "initiators": dict(initiators or {})} + return host_id + + def create_lun(self, name, size_bytes, tags): + self._record("create_lun", name, size_bytes) + all_tags = self.cluster_tags(tags, name) + with self._lock: + for lun in self.luns.values(): + if lun["name"] == name and lun["tags"].get("simplyblock-cluster-id") == self.cluster_id: + return {"id": lun["id"], "wwn": lun["wwn"], "size": lun["size"]} + type(self)._next += 1 + lun_id = f"lun-{self._next}" + type(self).luns[lun_id] = {"id": lun_id, "name": name, "wwn": f"wwn{self._next:04d}", + "size": int(size_bytes), "tags": all_tags, "hosts": []} + return {"id": lun_id, "wwn": f"wwn{self._next:04d}", "size": int(size_bytes)} + + def add_foreign_lun(self, lun_id, wwn="foreign", tags=None): + """A LUN on the array that this cluster did not create (tests).""" + with self._lock: + type(self).luns[lun_id] = {"id": lun_id, "name": lun_id, "wwn": wwn, "size": 1, + "tags": dict(tags or {}), "hosts": []} + + def attach(self, lun_id, host_id): + self._record("attach", lun_id, host_id) + with self._lock: + lun = self._lun(lun_id) + if host_id not in lun["hosts"]: + if len(lun["hosts"]) >= self.max_attachments: + raise ArrayError(f"fake array: {lun_id} is attached to {self.max_attachments} hosts") + lun["hosts"].append(host_id) + + def detach(self, lun_id, host_id, force=False): + self._record("detach", lun_id, host_id, force) + with self._lock: + lun = self._lun(lun_id) + if host_id in lun["hosts"]: + lun["hosts"].remove(host_id) + + def resize(self, lun_id, size_bytes): + self._record("resize", lun_id, size_bytes) + with self._lock: + lun = self._lun(lun_id) + if int(size_bytes) < lun["size"]: + raise ArrayError(f"fake array: {lun_id} cannot shrink") + lun["size"] = int(size_bytes) + + def get(self, lun_id): + with self._lock: + lun = self._lun(lun_id) + return {"id": lun["id"], "wwn": lun["wwn"], "size": lun["size"], + "hosts": list(lun["hosts"])} + + def delete_lun(self, lun_id): + self._record("delete_lun", lun_id) + with self._lock: + try: + self._lun(lun_id) + except ForeignLunError: + raise + except ArrayError: + return + type(self).luns.pop(lun_id, None) + + def list_luns(self): + with self._lock: + return [{"id": x["id"], "wwn": x["wwn"], "size": x["size"], "hosts": list(x["hosts"])} + for x in self.luns.values() + if x["tags"].get("simplyblock-cluster-id") == self.cluster_id] + + # fence / unfence: recorded as before (phase 3 tests fence LUNs by WWN + # that were never created here); applied to a LUN the fake knows. + + def _fence_op(self, op, lun, host): + with self._lock: + type(self).calls.append((op, lun, host)) + if op in self.fail: + raise ArrayError(f"fake array: {op} of {host} on {lun} failed") + if op in self.fail_falsy: + return False + with self._lock: + known = lun in self.luns or any(x["wwn"] == lun for x in self.luns.values()) + if known: + entry = self._lun(lun) + if op == "fence" and host in entry["hosts"]: + entry["hosts"].remove(host) + elif op == "unfence" and host not in entry["hosts"]: + entry["hosts"].append(host) + if op == "fence": + type(self).fenced.add((lun, host)) + else: + type(self).fenced.discard((lun, host)) + return True + + def fence(self, lun_id, host_id): + return self._fence_op("fence", lun_id, host_id) + + def unfence(self, lun_id, host_id): + return self._fence_op("unfence", lun_id, host_id) diff --git a/simplyblock_core/array/san.py b/simplyblock_core/array/san.py new file mode 100644 index 0000000000..ed5a68f700 --- /dev/null +++ b/simplyblock_core/array/san.py @@ -0,0 +1,349 @@ +"""SAN array plugins: the vendor adapters of :mod:`simplyblock_core.storage_arrays` +as :class:`~simplyblock_core.array.ArrayPlugin` s. + +One plugin per supported array family (``pure-flasharray``, ``netapp-ontap``, +... see :data:`SAN_PLUGINS`). Each wraps an +:class:`~simplyblock_core.storage_arrays.client.ArrayClient`, which already +gives the plugin contract its guarantees: idempotent create / map / unmap / +delete, grow-only resize, one re-authentication on a rejected session. + +Ownership. SAN arrays mostly have no free-form tags, so a LUN's owner is +encoded in its name: ``sb-`` (27 characters, +inside every supported array's name limit). The cluster part makes every +operation refuse a LUN of another cluster (:class:`ForeignLunError`); the LUN +part is derived from the name the control plane asked for, so a retried +create finds the LUN it made before. + +Hosts. A storage node is registered with the initiators of the protocol named +by the ``protocol`` option (``iscsi``, ``fc``, ``nvme-tcp``, ``nvme-rdma``, +``nvme-fc``, ``sdc``), taken from what its agent reports +(``host_initiators``: ``iscsi_iqn``, ``fc_wwpns``, ``nvme_hostnqn``, +``sdc_guid``). The host object is named ``sb-`` (shortened with a +hash suffix where the array's limit is shorter). + +Identity. ``wwn`` is returned in the form the hosts report it (lsblk ``WWN``): +the NAA hex digits for SCSI, ``eui.`` for NVMe namespaces. + +Options (``Cluster.array_options``): ``protocol`` (required), ``ca_bundle`` +(PEM), ``verify_tls`` (default true), ``timeout_s``, and the adapter's own +options (``option_docs`` of the adapter: pool, SVM, CPG, port group, ...). +``host_portals`` (iSCSI / NVMe discovery portals the nodes log in to) is read +by the node-side scan, not here. +""" + +from __future__ import annotations + +import hashlib +import logging +import re +import uuid as uuid_mod +from typing import Any + +from pydantic import SecretStr, ValidationError + +from simplyblock_core.array import ArrayError, ArrayPlugin, ForeignLunError +from simplyblock_core.credentials import Credential +from simplyblock_core.storage_arrays import base as sa +from simplyblock_core.storage_arrays.client import MANAGED_PREFIX, ArrayClient + +logger = logging.getLogger() + +#: Options consumed by the plugin itself; everything else goes to the adapter. +_PLUGIN_OPTIONS = frozenset({"protocol", "ca_bundle", "verify_tls", "timeout_s", + "host_portals"}) + +#: ``host_initiators`` keys per protocol. +_INITIATOR_KEYS = { + sa.Protocol.ISCSI: "iscsi_iqn", + sa.Protocol.FC: "fc_wwpns", + sa.Protocol.NVME_TCP: "nvme_hostnqn", + sa.Protocol.NVME_RDMA: "nvme_hostnqn", + sa.Protocol.NVME_FC: "nvme_hostnqn", + sa.Protocol.SDC: "sdc_guid", +} + +_CLUSTER_HEX = 8 +_LUN_HEX = 16 + + +def _hex(text: str, n: int) -> str: + return hashlib.sha256(text.encode()).hexdigest()[:n] + + +def lun_uuid(cluster_id: str, name: str) -> str: + """The volume UUID whose managed name encodes cluster and LUN name.""" + hex32 = _hex(f"cluster:{cluster_id}", _CLUSTER_HEX) + _hex(f"lun:{cluster_id}/{name}", 24) + return str(uuid_mod.UUID(hex32)) + + +def cluster_prefix(cluster_id: str) -> str: + return MANAGED_PREFIX + _hex(f"cluster:{cluster_id}", _CLUSTER_HEX) + + +def host_wwn(vol: sa.Volume, protocol: sa.Protocol) -> str: + """The identity a host reports for ``vol`` (lsblk ``WWN``).""" + if protocol.is_nvme and vol.nguid: + return vol.nguid.lower() + if vol.wwn: + w = vol.wwn.lower() + return w.removeprefix("naa.") + return vol.identifier.lower() + + +def host_object_name(hostname: str, max_len: int) -> str: + clean = re.sub(r"[^A-Za-z0-9-]", "-", hostname or "").strip("-") or "host" + name = f"{MANAGED_PREFIX}{clean}" + if len(name) <= max_len: + return name + suffix = "-" + _hex(hostname, 8) + return name[: max_len - len(suffix)].rstrip("-") + suffix + + +class _StaticCredentials: + """The plugin gets its credentials already resolved (from the KMS).""" + + def __init__(self, fields: dict[str, SecretStr]): + self._credential = Credential(fields) + + def resolve(self, ref, context): + return self._credential + + def invalidate(self, ref=None): + pass + + +def _as_list(value) -> list[str]: + if not value: + return [] + if isinstance(value, str): + return [v for v in re.split(r"[\s,]+", value) if v] + return [str(v) for v in value if v] + + +class SanArrayPlugin(ArrayPlugin): + """Base of the per-vendor plugins; ``vendor`` is the adapter key.""" + + vendor: str = "" + + def __init__(self, endpoint: str = "", options: dict | None = None, credentials=None, + cluster_id: str = ""): + super().__init__(endpoint, options, credentials, cluster_id) + self.client: ArrayClient | None = None + #: Tests: a ``requests`` transport (vendor fake) and session kwargs. + self.transport: Any = None + self.session_kwargs: dict[str, Any] = {} + + # ------------------------------------------------------------ setup + + @property + def protocol(self) -> sa.Protocol: + raw = str(self.options.get("protocol") or "").strip().lower() + if not raw: + raise ArrayError(f"array plugin {self.vendor}: option 'protocol' is required " + f"({', '.join(p.value for p in sa.Protocol)})") + try: + return sa.Protocol(raw) + except ValueError: + raise ArrayError(f"array plugin {self.vendor}: unknown protocol {raw!r}") from None + + def connect(self, endpoint: str, credentials: dict, options: dict) -> None: + super().connect(endpoint, credentials, options) + if not self.cluster_id: + raise ArrayError("the plugin has no cluster id") + proto = self.protocol + adapter_opts = {k: v for k, v in self.options.items() if k not in _PLUGIN_OPTIONS} + if self.vendor == "netapp-ontap": + # ONTAP creates SCSI LUNs or NVMe namespaces; which one follows + # from the protocol the hosts use, so it is derived, not repeated. + family = "nvme" if proto.is_nvme else "scsi" + stated = str(adapter_opts.get("protocol_family") or family).lower() + if stated != family: + raise ArrayError(f"netapp-ontap: protocol_family {stated!r} contradicts protocol " + f"{proto.value!r} (which needs {family!r})") + adapter_opts["protocol_family"] = family + try: + config = sa.ArrayConfig( + id=f"{self.cluster_id}", + vendor=self.vendor, + endpoint=endpoint, + # The plugin resolves nothing itself: the provider below hands + # back the credentials it was given. + credential_ref={"kind": "vault-kv", "mount": "-", "path": "-"}, + ca_bundle=self.options.get("ca_bundle") or None, + verify_tls=_truthy(self.options.get("verify_tls", True)), + timeout_s=float(self.options.get("timeout_s") or 60.0), + options=adapter_opts, + ) + except ValidationError as e: + raise ArrayError(f"array plugin {self.vendor}: {_first_error(e)}") from None + fields = {k: v if isinstance(v, SecretStr) else SecretStr(str(v)) + for k, v in (credentials or {}).items()} + self.client = ArrayClient(config, _StaticCredentials(fields), # type: ignore[arg-type] + transport=self.transport, session_kwargs=self.session_kwargs) + caps = self.client.capabilities + if proto not in caps.protocols: + raise ArrayError(f"{self.client.adapter_cls.display_name} does not support " + f"{proto.value} (supported: " + f"{', '.join(sorted(p.value for p in caps.protocols))})") + # Validates credential shape and options, logs in. + self._run(lambda c: c.get_info()) + + def _client(self) -> ArrayClient: + if self.client is None: + raise ArrayError("array plugin is not connected") + return self.client + + def _run(self, fn): + try: + return fn(self._client()) + except ArrayError: + raise + except sa.ArrayError as e: + raise ArrayError(str(e)) from e + + def capabilities(self) -> dict: + caps = self._client().capabilities if self.client else None + protocols = sorted(p.value for p in caps.protocols) if caps else [] + return {"protocols": protocols, "supports_reservations": False, "max_attachments": 0} + + # ---------------------------------------------------------- ownership + + def _prefix(self) -> str: + return cluster_prefix(self.cluster_id) + + def _owned_volume(self, lun_id: str) -> sa.Volume | None: + vol = self._run(lambda c: c.get_volume(lun_id)) + if vol is not None and not vol.name.startswith(self._prefix()): + raise ForeignLunError( + f"LUN {lun_id} ({vol.name}) was not created by simplyblock cluster " + f"{self.cluster_id}; it is not touched") + return vol + + def _view(self, vol: sa.Volume) -> dict: + hosts = [m.host_id for m in self._run(lambda c: c.list_mappings(vol.id))] + return {"id": vol.id, "wwn": host_wwn(vol, self.protocol), "size": vol.size_bytes, + "name": vol.name, "hosts": hosts} + + # --------------------------------------------------------- operations + + def ensure_host(self, name: str, initiators: dict) -> str: + proto = self.protocol + key = _INITIATOR_KEYS[proto] + addresses = _as_list((initiators or {}).get(key)) + if not addresses: + raise ArrayError( + f"host {name!r} reports no {proto.value} initiator ({key}); check the " + "host's initiator setup (iSCSI initiatorname, FC HBA, NVMe hostnqn, SDC)") + try: + spec = sa.HostSpec( + name=host_object_name(name, self._client().capabilities.max_name_len), + initiators=tuple(sa.Initiator(protocol=proto, address=a) for a in addresses)) + except ValidationError as e: + raise ArrayError(f"host {name!r}: {_first_error(e)}") from None + return self._run(lambda c: c.ensure_host(spec)) + + def create_lun(self, name: str, size_bytes: int, tags: dict) -> dict: + self.cluster_tags(tags, name) # refuses without a cluster id + if int(size_bytes) <= 0: + raise ArrayError("a LUN needs a size") + vol = self._run(lambda c: c.create_volume(lun_uuid(self.cluster_id, name), int(size_bytes))) + return {"id": vol.id, "wwn": host_wwn(vol, self.protocol), "size": vol.size_bytes} + + def attach(self, lun_id: str, host_id: str) -> None: + if self._owned_volume(lun_id) is None: + raise ArrayError(f"LUN {lun_id} not found") + self._run(lambda c: c.map_volume(lun_id, host_id)) + + def detach(self, lun_id: str, host_id: str, force: bool = False) -> None: + if self._owned_volume(lun_id) is None: + return + # An unmap at the array takes effect for the host at once, queued + # I/O included: the same with and without ``force``. + self._run(lambda c: c.unmap_volume(lun_id, host_id)) + + def resize(self, lun_id: str, size_bytes: int) -> None: + if self._owned_volume(lun_id) is None: + raise ArrayError(f"LUN {lun_id} not found") + self._run(lambda c: c.resize_volume(lun_id, int(size_bytes))) + + def get(self, lun_id: str) -> dict: + vol = self._owned_volume(lun_id) + if vol is None: + raise ArrayError(f"LUN {lun_id} not found") + return self._view(vol) + + def delete_lun(self, lun_id: str) -> None: + if self._owned_volume(lun_id) is None: + return + self._run(lambda c: c.delete_volume(lun_id)) + + def list_luns(self) -> list[dict]: + """Not offered by the SAN adapters (no portable prefix listing); + ``array_controller.delete_cluster_luns`` deletes the recorded LUNs.""" + raise NotImplementedError(f"{self.vendor}: listing LUNs is not supported") + + +def _truthy(v) -> bool: + if isinstance(v, bool): + return v + return str(v).strip().lower() not in ("0", "false", "no", "off", "") + + +def _first_error(e: ValidationError) -> str: + err = e.errors()[0] + loc = ".".join(str(p) for p in err.get("loc", ())) + return f"{loc}: {err.get('msg')}" if loc else str(err.get("msg")) + + +class PureFlashArrayPlugin(SanArrayPlugin): + vendor = "pure-flasharray" + + +class NetAppOntapPlugin(SanArrayPlugin): + vendor = "netapp-ontap" + + +class DellPowerStorePlugin(SanArrayPlugin): + vendor = "dell-powerstore" + + +class DellPowerMaxPlugin(SanArrayPlugin): + vendor = "dell-powermax" + + +class DellPowerFlexPlugin(SanArrayPlugin): + vendor = "dell-powerflex" + + +class HpeWsapiPlugin(SanArrayPlugin): + vendor = "hpe-wsapi" + + +class HpeNimblePlugin(SanArrayPlugin): + vendor = "hpe-nimble" + + +class IbmFlashSystemPlugin(SanArrayPlugin): + vendor = "ibm-flashsystem" + + +class HitachiVspPlugin(SanArrayPlugin): + vendor = "hitachi-vsp" + + +class HuaweiDoradoPlugin(SanArrayPlugin): + vendor = "huawei-dorado" + + +class InfinidatInfiniBoxPlugin(SanArrayPlugin): + vendor = "infinidat-infinibox" + + +#: Plugin name (= adapter key) -> import path, merged into BUILTIN_PLUGINS. +SAN_PLUGINS = { + cls.vendor: f"{__name__}:{cls.__name__}" + for cls in (PureFlashArrayPlugin, NetAppOntapPlugin, DellPowerStorePlugin, + DellPowerMaxPlugin, DellPowerFlexPlugin, HpeWsapiPlugin, HpeNimblePlugin, + IbmFlashSystemPlugin, HitachiVspPlugin, HuaweiDoradoPlugin, + InfinidatInfiniBoxPlugin) +} diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index f154629c80..26e0e93388 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -34,6 +34,7 @@ import docker from simplyblock_core import ( constants, + floors, index_ops, jm_raid, mgmt_node_ops, @@ -44,6 +45,7 @@ ) from simplyblock_core.controllers import ( activation_gate, + array_controller, cluster_events, device_controller, pool_controller, @@ -56,6 +58,9 @@ from simplyblock_core.controllers.cluster_expansion.planner import ( RemoteTripletPlacementError, ) +from simplyblock_core.controllers.cluster_expansion.planner import ( + fd_activation_min_domains as fd_planner_min_domains, +) from simplyblock_core.db_controller import DBController from simplyblock_core.exceptions import ( MonitoringConfigError, @@ -425,6 +430,55 @@ def _validate_leader_site(sync_replication, leader_site) -> None: if leader_site and not sync_replication: raise SyncReplicationSiteError( "leader_site is only allowed together with sync_replication") +#: Erasure-coding scheme a multi-attach (array LUN) cluster runs: the array +#: already protects every LUN, so simplyblock adds no parity. +MULTI_ATTACH_SCHEME = (1, 0) + + +def _resolve_multi_attach(multi_attach, ftt, raid_devices, ha_type, is_single_node, + distr_ndcs, distr_npcs, max_fault_tolerance): + """Validate the multi-attach array LUN options and return the effective + ``(distr_ndcs, distr_npcs, max_fault_tolerance)``. + + Without ``multi_attach`` the inputs are returned unchanged (``ftt`` must + then be unset). With it the cluster runs 1+0 (the array protects the + LUNs) and tolerates ``ftt`` node failures by reopening an owner's LUN on + another node, so it needs ``raid_devices``, an HA cluster (the LUN is + attached to the LVS secondary/tertiary) and more than one node. + + Raises: + ValueError: an inconsistent combination of options. + """ + if not multi_attach: + if ftt is not None: + raise ValueError("ftt is only valid together with multi_attach") + return distr_ndcs, distr_npcs, max_fault_tolerance + if not raid_devices: + raise ValueError("multi_attach requires raid_devices: the array must protect every LUN") + if ftt not in (1, 2): + raise ValueError("multi_attach requires ftt 1 or 2") + if ha_type != "ha": + raise ValueError("multi_attach requires ha_type='ha'") + if is_single_node: + raise ValueError("multi_attach cannot be used on a single-node cluster") + if (distr_ndcs, distr_npcs) != MULTI_ATTACH_SCHEME: + logger.info("multi_attach: erasure coding %s+%s overridden by %s+%s", + distr_ndcs, distr_npcs, *MULTI_ATTACH_SCHEME) + return MULTI_ATTACH_SCHEME[0], MULTI_ATTACH_SCHEME[1], int(ftt) + + +def _validate_warm_standby(multi_attach, warm_standby): + """Warm standbys (Cluster.multi_attach_warm_standby) are a multi-attach + option only.""" + if warm_standby and not multi_attach: + raise ValueError("multi_attach_warm_standby requires multi_attach") + if warm_standby and not constants.MULTI_ATTACH_WARM_STANDBY_ALLOWED: + raise ValueError( + "multi_attach_warm_standby is disabled: on the 2026-11-20 bed a takeover onto a warm " + "standby returned stale data (fio crc32c verify failures); it stays off until the " + "alceml warm refresh is fixed") + + def _validate_integrity_flags(inline_checksum, atomic_4k): """atomic_4k only lifts the data plane's 4K gate on fallback-mode checksums, so without inline_checksum it is a setting that does nothing.""" @@ -452,7 +506,15 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, sync_replication=False, page_copy=False, md_journal=True, + raid_devices=False, + multi_attach=False, + ftt=None, + multi_attach_warm_standby=False, ) -> str: + distr_ndcs, distr_npcs, max_fault_tolerance = _resolve_multi_attach( + multi_attach, ftt, raid_devices, ha_type, is_single_node, + distr_ndcs, distr_npcs, max_fault_tolerance) + _validate_warm_standby(multi_attach, multi_attach_warm_standby) if (distr_ndcs, distr_npcs) not in SUPPORTED_ERASURE_CODING_SCHEMES: raise ValueError("Unsupported erasure coding scheme") _validate_integrity_flags(inline_checksum, atomic_4k) @@ -462,7 +524,9 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, if max_fault_tolerance > 1: if ha_type != "ha": raise ValueError("max_fault_tolerance > 1 requires ha_type='ha'") - if distr_npcs < 2: + # Multi-attach runs 1+0: its FTT comes from the array LUN being + # reopened on another node, not from parity. + if distr_npcs < 2 and not multi_attach: raise ValueError("max_fault_tolerance > 1 requires distr_npcs >= 2") if kms_settings is not None: @@ -621,6 +685,9 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, cluster.atomic_4k = bool(atomic_4k) cluster.page_copy = bool(page_copy) cluster.md_journal = bool(md_journal) + cluster.raid_devices = bool(raid_devices) + cluster.multi_attach = bool(multi_attach) + cluster.multi_attach_warm_standby = bool(multi_attach and multi_attach_warm_standby) cluster.client_data_nic = client_data_nic or "" cluster.max_fault_tolerance = max_fault_tolerance cluster.nvmf_base_port = nvmf_base_port @@ -775,6 +842,10 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn leader_site="", page_copy=False, md_journal=True, + raid_devices=False, + multi_attach=False, + ftt=None, + multi_attach_warm_standby=False, snode_api_service=constants.K8S_SNODE_API_SERVICE, spdk_proxy_service=constants.K8S_SPDK_PROXY_SERVICE, spdk_proxy_tls_secret=constants.K8S_SPDK_PROXY_TLS_SECRET, @@ -814,6 +885,10 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn leader_site=leader_site, page_copy=page_copy, md_journal=md_journal, + raid_devices=raid_devices, + multi_attach=multi_attach, + ftt=ftt, + multi_attach_warm_standby=multi_attach_warm_standby, snode_api_service=snode_api_service, spdk_proxy_service=spdk_proxy_service, spdk_proxy_tls_secret=spdk_proxy_tls_secret, @@ -853,6 +928,10 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca leader_site="", page_copy=False, md_journal=True, + raid_devices=False, + multi_attach=False, + ftt=None, + multi_attach_warm_standby=False, snode_api_service=constants.K8S_SNODE_API_SERVICE, spdk_proxy_service=constants.K8S_SPDK_PROXY_SERVICE, spdk_proxy_tls_secret=constants.K8S_SPDK_PROXY_TLS_SECRET, @@ -867,6 +946,10 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca if existing.cluster_name and existing.cluster_name == name: raise ValueError(f"A cluster with the name '{name}' already exists") + distr_ndcs, distr_npcs, max_fault_tolerance = _resolve_multi_attach( + multi_attach, ftt, raid_devices, ha_type, is_single_node, + distr_ndcs, distr_npcs, max_fault_tolerance) + _validate_warm_standby(multi_attach, multi_attach_warm_standby) if (distr_ndcs, distr_npcs) not in SUPPORTED_ERASURE_CODING_SCHEMES: raise ValueError("Unsupported erasure coding scheme") @@ -876,7 +959,9 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca if max_fault_tolerance > 1: if ha_type != "ha": raise ValueError("max_fault_tolerance > 1 requires ha_type='ha'") - if distr_npcs < 2: + # Multi-attach runs 1+0: its FTT comes from the array LUN being + # reopened on another node, not from parity. + if distr_npcs < 2 and not multi_attach: raise ValueError("max_fault_tolerance > 1 requires distr_npcs >= 2") if kms_settings is not None: @@ -1003,6 +1088,9 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca cluster.atomic_4k = bool(atomic_4k) cluster.page_copy = bool(page_copy) cluster.md_journal = bool(md_journal) + cluster.raid_devices = bool(raid_devices) + cluster.multi_attach = bool(multi_attach) + cluster.multi_attach_warm_standby = bool(multi_attach and multi_attach_warm_standby) cluster.client_data_nic = client_data_nic or "" cluster.max_fault_tolerance = max_fault_tolerance cluster.nvmf_base_port = nvmf_base_port @@ -1062,6 +1150,25 @@ def mutate(fresh): db_controller.atomic_update(cluster, mutate) +def set_array_plugin(cl_id, plugin: str, endpoint: str = "", options: dict | None = None, + credentials: dict | None = None, lun_size: int = 0) -> None: + """Configure the storage-array plugin of a multi-attach cluster (see + controllers/array_controller.register_array: the plugin is connected + before anything is saved, credentials go to the KMS). An empty + ``plugin`` removes the registration, after which takeovers are refused. + + Raises: + ValueError: not a multi-attach cluster, the plugin cannot be loaded + or connected, or (removal) LUNs created through it exist. + """ + cluster = db_controller.get_cluster_by_id(cl_id) + if not plugin: + if cluster.array_plugin: + array_controller.delete_array(cl_id, cluster.array_id) + return + array_controller.register_array(cl_id, plugin, endpoint, options, credentials, lun_size) + + def _wait_for_full_device_connectivity(cl_id, timeout_sec=300, poll_sec=10, checkpoint=None): """Block until every ONLINE primary node holds a connected remote-device record for every ONLINE device of every OTHER online node, or raise. @@ -1497,12 +1604,23 @@ def _record_activation_topology(cluster, single_node, fresh) -> None: def activation_minimum_devices(cluster, single_node_cluster) -> int: - """ndcs+npcs devices are needed for placement; the +1 spare is rebuild - headroom that a single-node cluster (no data redundancy to rebuild onto - a spare) does not require.""" + """The RE-activation device gate (recovery of an existing layout): ndcs+npcs + devices are needed for placement; the +1 spare is rebuild headroom that a + single-node cluster (no data redundancy to rebuild onto a spare) does not + require. A FRESH activation is judged by the full floors instead + (``activation_floor_violations``); a re-activation deliberately keeps + this weaker gate so a drifted cluster can still be recovered.""" return cluster.distr_ndcs + cluster.distr_npcs + (0 if single_node_cluster else 1) +def activation_floor_violations(cluster, snodes, online_nodes) -> builtins.list[str]: + """The floors (simplyblock_core.floors) a fresh activation would violate + with ``online_nodes`` online: node count, one data device per node, the + cluster-wide data-device count and, with failure domains, the domain + count. Journal devices never count (they sit in ``jm_device``).""" + return floors.evaluate(cluster, snodes, online_nodes=online_nodes) + + #: Sites of a sync-replication cluster: one simplyblock cluster spans exactly two. SYNC_SITE_COUNT = 2 #: Distinct hosts a site needs for a host-disjoint primary/secondary/tertiary triplet. @@ -1735,10 +1853,23 @@ def _revert() -> None: logger.warning("Single-node cluster: activating as non-HA " "(no secondary nodes, single journal) regardless of ha_type") - minimum_devices = activation_minimum_devices(cluster, single_node_cluster) - if dev_count < minimum_devices: - _revert() - raise ValueError(f"Failed to activate cluster, No enough online device.. Minimum is {minimum_devices}") + if is_fresh_activation: + # Fresh activation refuses below any floor (simplyblock_core.floors). + # A node without a data device is refused here too: downstream, + # _prepare_cluster_devices_jm_on_dev accepts an empty device list. + floor_violations = activation_floor_violations(cluster, snodes, online_nodes) + if floor_violations: + _revert() + raise ValueError("Failed to activate cluster, below the resource floors for " + f"the {cluster.distr_ndcs}+{cluster.distr_npcs} scheme: " + + "; ".join(floor_violations)) + else: + # Re-activation (recovery): the weaker device gate of before, so a + # drifted or partially-online cluster can still be brought back. + minimum_devices = activation_minimum_devices(cluster, single_node_cluster) + if dev_count < minimum_devices: + _revert() + raise ValueError(f"Failed to activate cluster, No enough online device.. Minimum is {minimum_devices}") # Sync replication: the configured two-site layout is enforced on every # activation (force included) -- a third site or an unsited node is never @@ -1784,7 +1915,7 @@ def _revert() -> None: fd_desired_layout: dict[str, tuple[str, str]] = {} if cluster.enable_failure_domain: distinct_domains = {node.failure_domain for node in online_nodes if node.failure_domain >= 0} - min_domains = cluster.distr_npcs + 2 + min_domains = fd_planner_min_domains(cluster.distr_npcs, cluster.distr_ndcs) if len(distinct_domains) < min_domains: logger.warning( "Failure-domain feature is enabled but only %d distinct failure " @@ -1841,7 +1972,7 @@ def _on_site(msg: str, site=site) -> str: # stranding another node's secondary/tertiary with none at all. # This also subsumes the plain "at least two domains" floor. domain_count_violation = fd_planner.fd_activation_domain_count_violation( - cluster.distr_npcs, len(fd_host_counts)) + cluster.distr_npcs, len(fd_host_counts), ndcs=cluster.distr_ndcs) if domain_count_violation: _fd_fail(_on_site(domain_count_violation)) if len(set(fd_host_counts.values())) != 1: @@ -1985,6 +2116,10 @@ def _on_site(msg: str, site=site) -> str: snode = db_controller.get_storage_node_by_id(snode.get_id()) used_nodes_as_tertiary.append(snode.tertiary_node_id) + # Multi-attach cluster with an array: present every owner's LUN to + # its new secondary / tertiary (a no-op on every other cluster). + array_controller.reconcile_after_role_change(cl_id) + # Sync replication: every LVS also gets its remote triplet on the other # site. Refs a re-activation finds are kept; missing ones are picked. if cluster.sync_replication: @@ -2771,6 +2906,10 @@ def cluster_expand(cl_id) -> None: sec_node_2.lvstore_stack_tertiary = snode.get_id() sec_node_2.write_to_db() + if cluster.ha_type == "ha": + # Multi-attach cluster with an array: the new roles' LUN paths. + array_controller.reconcile_after_role_change(cl_id) + if cluster.ha_type == "ha" and cluster.sync_replication: try: storage_node_ops.assign_remote_triplets(cluster, [snode.get_id()]) @@ -4516,6 +4655,10 @@ def delete_cluster(cl_id, force=False) -> None: if pools: raise ValueError("Can only remove Empty cluster, Pools found") + # Multi-attach cluster with an array: the only place its LUNs are + # deleted, with the array credentials in the KMS. No-op otherwise. + array_controller.delete_cluster_luns(cluster, force=force) + logger.info(f"Deleting Cluster {cl_id}") cluster_events.cluster_delete(cluster) cluster.remove(db_controller.kv_store) @@ -4557,7 +4700,18 @@ def set_(cl_id, attr, value) -> bool: cluster.write_to_db() return True else: - value = cluster.get_attrs_map()[attr]['type'](value) + attr_type = cluster.get_attrs_map()[attr]['type'] + if attr_type is bool and isinstance(value, str): + # bool("false") is True; parse the CLI's string form explicitly. + lowered = value.strip().lower() + if lowered in ("1", "true", "yes", "on"): + value = True + elif lowered in ("0", "false", "no", "off"): + value = False + else: + raise ValueError(f"Not a boolean value for {attr}: {value!r}") + else: + value = attr_type(value) logger.info(f"Setting {attr} to {value}") if attr == "two_node_arbitration": set_two_node_arbitration(cl_id, value) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 971c831f28..a71393ff9d 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -129,6 +129,29 @@ def get_config_var(name, default=None): # Queue-depth sampling period enabled on every AIO base bdev so # bdev_get_iostat reports queue_depth (feeds the hung-IO watchdog). AIO_QD_SAMPLING_PERIOD_US = 100000 # 100 ms +# Block size of the AIO base bdev of a NEW lblk device. 512e devices (512 +# logical / 4096 physical, e.g. EBS and most array LUNs) would otherwise get +# 512-byte bdevs, which costs read-modify-write cycles in the device. The +# logical block size is used only when 4096 is not possible (see +# utils.choose_aio_block_size). An existing device always keeps the block +# size it was formatted with (NVMeDevice.aio_block_size): alceml's on-disk +# layout depends on it. +AIO_DEFAULT_BLOCK_SIZE = 4096 +# I/O engine of an lblk device's base bdev (NVMeDevice.lblk_io_engine). +# SPDK's bdev_aio submits every I/O on a block device with RWF_NOWAIT so +# io_submit never blocks the reactor. device-mapper honours RWF_NOWAIT only +# when every target in the table does; dm-multipath does not, so every AIO +# submit to a multipath map fails with EOPNOTSUPP. Device-mapper devices are +# therefore served by SPDK's io_uring bdev (bdev_uring_create), everything +# else (whole disks, sd/nvme partitions, md arrays) by bdev_aio. The bdev +# NAME (aio_) and NVMeDevice.bdev_type "aio" are the same for both. +LBLK_IO_ENGINE_AIO = "aio" +LBLK_IO_ENGINE_URING = "uring" +LBLK_IO_ENGINES = (LBLK_IO_ENGINE_AIO, LBLK_IO_ENGINE_URING) +# lsblk TYPE values of device-mapper nodes (multipath map, LVM volume, +# dm-crypt, other dm). A partition mapped from a multipath map (kpartx) has +# TYPE "part" but a dm-* kernel name. +LBLK_DM_TYPES = ("mpath", "lvm", "crypt", "dm") # Hung-IO watchdog: consecutive device_monitor polls (DEV_MONITOR_INTERVAL_SEC # apart) with queue_depth > 0 and zero completion progress before the device # is declared stalled (3 x 10s = 30s — deliberately above kernel SCSI/NVMe @@ -1294,6 +1317,88 @@ def get_config_var(name, default=None): qos_class_replication_weight_percent = 10 MIG_PARALLEL_JOBS = 64 +# Send the placement anti-affinity floor to the data plane (affinity_floor on +# distr_migration_*_start, allow_device_colocation on bdev_distrib_create). +# Flip to True once the ultra feat/affinity-floor data plane is deployed; +# older data planes reject unknown RPC parameters. +DISTR_AFFINITY_FLOOR_RPC = False + +# Multi-attach array LUN mode (cluster.multi_attach). None of these apply to +# a regular cluster. +# Peers' controllers to a multi-attach device hold I/O this long while no +# path is available (per-controller ctrlr_loss_timeout_sec), so a takeover +# that adds the LUN's new export as a path within the window resumes I/O +# instead of failing it. The cluster-wide bdev_nvme defaults (1 s) stay. +# It must cover the slowest takeover: the regular monitor's detection of the +# node (~20 s) when the fast probe (MULTI_ATTACH_PROBE_*) did not trigger, and +# the takeover up to its MULTI_ATTACH_TAKEOVER_DEADLINE_SEC. The I/O stall of a +# takeover started by the fast probe is a few seconds. +MULTI_ATTACH_HOLD_SEC = 150 +MULTI_ATTACH_RECONNECT_DELAY_SEC = 2 +MULTI_ATTACH_FAST_IO_FAIL_SEC = MULTI_ATTACH_HOLD_SEC +# The alceml of a multi-attach device fails its pending I/O and unregisters +# after this long without completion (bdev_alceml_create stuck_io_timeout_ms; +# the data-plane default is 4000 ms). It must exceed the multipath path +# failover below the LUN and stay below MULTI_ATTACH_HOLD_SEC, so a stuck +# owner instance is gone before the peers stop waiting. +MULTI_ATTACH_ALCEML_STUCK_IO_MS = 60000 +# Controller-ID ranges of multi-attach device subsystems: a node with +# cntlid_block i exports with cntlids [i*64+1, i*64+64]. SPDK multipath needs +# distinct cntlids across the paths of one controller, and a LUN may be +# exported by any node. Valid NVMe cntlids end at 0xFFEF. +MULTI_ATTACH_CNTLID_BLOCK_SIZE = 64 +MULTI_ATTACH_MAX_CNTLID = 0xFFEF +MULTI_ATTACH_MAX_NODES = 1022 +# A LUN takeover (controllers/lun_takeover) must finish well inside the +# peers' hold, or their I/O fails before the new path exists; it gives up at +# this deadline (seconds from its start) and the device goes unavailable. +MULTI_ATTACH_TAKEOVER_DEADLINE_SEC = 90 +# Fence a lost node's LUN with an NVMe/SCSI persistent reservation instead of +# waiting for the array to detach it: every multi-attach alceml registers a +# key (multi_attach.reservation_key), the owner holds the reservation, and a +# takeover's alceml preempts it at open (ultra #236), in milliseconds. The +# array fence (EBS force-detach: 0.5-4 s, longer with several LUNs of one +# node) then follows off the I/O path. Needs an array plugin that reports +# supports_reservations and an ultra data plane with LUN failover. +MULTI_ATTACH_RESERVATIONS = True +# A takeover waits this long for its alceml to report the reservation taken. +MULTI_ATTACH_ACTIVATION_WAIT_SEC = 5 +# Warm standby (Cluster.multi_attach_warm_standby): ms between two refreshes +# of a standby alceml's page map from the LUN's region counters (ultra #236, +# refresh_interval_ms; the data plane's default is the same). +MULTI_ATTACH_WARM_REFRESH_MS = 2000 +# How often the storage-node monitor reconciles a node's warm standbys +# (lun_takeover.ensure_warm_standbys), per node. +MULTI_ATTACH_WARM_STANDBY_INTERVAL_SEC = 30 +# Warm standby is refused at cluster create: on the 2026-11-20 FTT2 bed a +# takeover onto a warm standby (activation 141 ms) was followed by crc32c +# verify failures on 4 of 5 volumes; the cold takeover on the same build and +# case verified clean. Test beds that reproduce it set +# SIMPLYBLOCK_ALLOW_WARM_STANDBY=1 on the control plane. +MULTI_ATTACH_WARM_STANDBY_ALLOWED = os.environ.get("SIMPLYBLOCK_ALLOW_WARM_STANDBY") == "1" +# Client keep-alive on multi-attach clusters: a node lost without closing its +# connections (hard reset, network cut) leaves its own volumes' paths dead +# until the host's keep-alive notices, 1-2 intervals: 8 s took 10-14 s on the +# bed. 3 s bounds that to 3-6 s. +MULTI_ATTACH_CLIENT_KEEP_ALIVE_TO = 3 +# Fast loss detection of a node serving LUNs (services/lun_watchdog). With +# 1+0 every volume waits on a lost node's LUN until its takeover, so the +# takeover starts from this probe, not from the node's status: the OFFLINE +# verdict took minutes on the multi-attach bed. The node's SPDK RPC is probed +# every PROBE_INTERVAL with PROBE_TIMEOUT, together with its node agent. A +# node is lost when its agent reports SPDK not running (at once), or when +# neither SPDK nor the agent has answered for LOST_AFTER. The fence comes +# first, so a takeover of a node that is alive behind a management network +# fault is safe; it costs a short I/O pause on its LUNs. +MULTI_ATTACH_PROBE_INTERVAL_SEC = 0.5 +MULTI_ATTACH_PROBE_TIMEOUT_SEC = 0.8 +MULTI_ATTACH_LOST_AFTER_SEC = 2.0 +# The array connection a fence uses is kept this long (a new one validates +# the credentials first: about a second on EBS). +ARRAY_PLUGIN_CACHE_SEC = 600 +# Lease of a running takeover (one per LUN, in FDB): a holder older than this +# is presumed dead. +MULTI_ATTACH_TAKEOVER_LOCK_TTL_SEC = 2 * MULTI_ATTACH_TAKEOVER_DEADLINE_SEC MIG_JOB_SIZE = 64 # Sync-replication resync (tasks_runner_sync_resync): the shared task driver diff --git a/simplyblock_core/controllers/alerts_controller.py b/simplyblock_core/controllers/alerts_controller.py index 0607d4f2f4..48edbdb79d 100644 --- a/simplyblock_core/controllers/alerts_controller.py +++ b/simplyblock_core/controllers/alerts_controller.py @@ -72,6 +72,7 @@ CLUSTER_PROV_CAPACITY_CRITICAL = "cluster_provisioned_capacity_critical" DEVICE_SLOW = "device_slow" DEVICE_UNHEALTHY = "device_unhealthy" +DEVICE_SHRUNK = "device_shrunk" SEVERITY_CRITICAL = "critical" SEVERITY_WARNING = "warning" @@ -290,6 +291,19 @@ def _device_alerts(cluster_id, nodes) -> list[dict]: label = _node_label(node) for dev in (node.nvme_devices or []): + # A block device (LUN) seen smaller than its storage device by + # the last rescan (device_controller.grow_cluster_devices). It is + # a state: it resolves when a rescan sees the restored size. + backend_size = getattr(dev, "backend_size", 0) or 0 + if 0 < backend_size < (getattr(dev, "size", 0) or 0): + alerts.append(_alert( + DEVICE_SHRUNK, SEVERITY_CRITICAL, + f"block device of device {dev.get_id()} on node {label} shrank to " + f"{utils.humanbytes(backend_size)}, below the device size of " + f"{utils.humanbytes(dev.size)}: restore the LUN size", + cluster_id, node_id=node.get_id(), device_id=dev.get_id(), + device_status=dev.status, + serial_number=getattr(dev, "serial_number", ""))) # Removed counts: a device that vanished from an otherwise # healthy node is a failure or a physical pull, and either way # somebody has to look. The one removal that is not is the diff --git a/simplyblock_core/controllers/array_controller.py b/simplyblock_core/controllers/array_controller.py new file mode 100644 index 0000000000..5b294db486 --- /dev/null +++ b/simplyblock_core/controllers/array_controller.py @@ -0,0 +1,962 @@ +"""Multi-attach array LUN mode, phase 4: the storage array of a cluster. + +simplyblock creates, presents, grows, fences and deletes the LUNs of a +multi-attach cluster itself, through the cluster's array plugin +(simplyblock_core.array). This module holds: + +* registration (``register_array`` / ``get_arrays`` / ``delete_array``): the + plugin is loaded, connected and asked for its capabilities before anything + is saved; credentials go to the KMS at ``cluster//array/``, + plugin name, endpoint and options to the Cluster record. One array per + cluster. +* the LUN lifecycle, every hook a no-op unless the cluster is multi-attach + and has an array plugin (``array_configured``), and acting only on LUNs the + plugin created (``NVMeDevice.array_lun_id``): + + - node add: ``provision_node_lun`` creates the node's LUN + (``Cluster.array_lun_size``), registers the host, attaches the LUN and + waits for it in the node agent's block-device inventory; the add then + continues as a regular lblk add with that device. With the journal on + the array (``Cluster.array_journal`` "lun") it first creates the node's + journal LUN the same way, with the journal options (its pool), and + presents it to that node only. + - LVS roles: ``reconcile_lun_attachments`` presents every owner's LUN to + its secondary and tertiary (and the node serving it), records the path + there in ``node_paths``, and detaches it from a node that lost its role + -- never from the serving node, never to a node fenced from it. + - growth: ``grow_luns`` grows every LUN by one factor at the array and + waits until every node it is presented to sees the new size. + - node delete: ``release_node_luns``; cluster delete: + ``delete_cluster_luns`` (the only place a LUN is deleted). +""" + +from __future__ import annotations + +import logging +import math +import threading +import time +import uuid as uuid_mod + +from pydantic import SecretStr + +from simplyblock_core import array, multi_attach + +logger = logging.getLogger() + +#: How long a node add waits for its new LUN to show in the agent inventory. +LUN_APPEAR_TIMEOUT_SEC = 120 +#: How long the attachment reconcile waits for a LUN on a newly attached node. +LUN_PATH_TIMEOUT_SEC = 30 +#: How long a growth waits for every node to see the new size. +LUN_GROW_VISIBLE_TIMEOUT_SEC = 180 +POLL_SEC = 3 + +#: Cluster.array_journal: the journal on a local disk or partition of the +#: node's host, or on a LUN of its own on the array. +JOURNAL_LOCAL = "local" +JOURNAL_LUN = "lun" +JOURNAL_PLACEMENTS = (JOURNAL_LOCAL, JOURNAL_LUN) +#: A journal LUN of no given size: this share of the data LUN, at least +#: JOURNAL_LUN_MIN_BYTES, rounded up to whole GiB. +JOURNAL_LUN_PERCENT = 3 +JOURNAL_LUN_MIN_BYTES = 2 << 30 + +_STATUS_REMOVED = "removed" +_STATUS_ONLINE = "online" + + +class ArrayLifecycleError(Exception): + """A LUN lifecycle step failed; the message says which.""" + + +def _sleep(sec): + time.sleep(sec) + + +def _db(): + from simplyblock_core.db_controller import DBController + return DBController() + + +def array_configured(cluster) -> bool: + """A multi-attach cluster with an array plugin: the lifecycle hooks act.""" + return multi_attach.is_multi_attach(cluster) and bool(getattr(cluster, "array_plugin", "")) + + +def provisions_luns(cluster) -> bool: + """Node adds create the node's LUN (an array is configured and a LUN + size set).""" + return array_configured(cluster) and int(getattr(cluster, "array_lun_size", 0) or 0) > 0 + + +def journal_on_lun(cluster) -> bool: + """Node adds also create the node's journal LUN (``Cluster.array_journal`` + "lun" on a cluster that creates the nodes' LUNs).""" + return provisions_luns(cluster) and (getattr(cluster, "array_journal", "") or JOURNAL_LOCAL) == JOURNAL_LUN + + +def journal_lun_size(cluster) -> int: + """Size in bytes of a node's journal LUN.""" + size = int(getattr(cluster, "array_journal_lun_size", 0) or 0) + if size: + return size + size = max(JOURNAL_LUN_MIN_BYTES, math.ceil(int(cluster.array_lun_size or 0) * JOURNAL_LUN_PERCENT / 100)) + return math.ceil(size / (1 << 30)) << 30 + + +def _event(cluster_id, message, error=False): + try: + from simplyblock_core.controllers import events_controller as ec + from simplyblock_core.models.events import EventObj + ec.log_event_cluster( + cluster_id=cluster_id, domain=ec.DOMAIN_CLUSTER, event=ec.EVENT_ARRAY_LUN, + db_object=_db().get_cluster_by_id(cluster_id), caused_by=ec.CAUSED_BY_MONITOR, + message=message, event_level=EventObj.LEVEL_ERROR if error else EventObj.LEVEL_INFO) + except Exception: # noqa: BLE001 - an event is best effort + logger.exception("could not log the array event") + + +# ------------------------------------------------------------ registration + +def _wrap_credentials(credentials) -> dict: + out = {} + for k, v in (credentials or {}).items(): + if not isinstance(k, str) or not k: + raise ValueError("credential names must be non-empty strings") + if isinstance(v, SecretStr): + out[k] = v + elif isinstance(v, str): + out[k] = SecretStr(v) + else: + raise ValueError(f"credential {k!r} must be a string") + return out + + +def _managed_devices(nodes): + """``(owner, device)`` of every LUN the array plugin created.""" + return [(n, d) for n in nodes for d in (n.nvme_devices or []) + if multi_attach.is_multi_attach(d) and d.array_lun_id] + + +def array_view(cluster, capabilities=None) -> dict: + """The registered array of ``cluster`` as the API shows it: everything + but the credentials (only whether there are any).""" + out = { + "id": cluster.array_id, + "plugin": cluster.array_plugin, + "endpoint": cluster.array_endpoint, + "options": dict(cluster.array_options or {}), + "lun_size": int(cluster.array_lun_size or 0), + "journal": getattr(cluster, "array_journal", "") or JOURNAL_LOCAL, + "journal_lun_size": (journal_lun_size(cluster) + if (getattr(cluster, "array_journal", "") or JOURNAL_LOCAL) == JOURNAL_LUN + and int(cluster.array_lun_size or 0) else 0), + "journal_options": dict(getattr(cluster, "array_journal_options", None) or {}), + "has_credentials": bool(cluster.array_has_credentials), + "credential_ref": array.credential_ref_view(cluster), + } + if capabilities is not None: + out["capabilities"] = capabilities + return out + + +def register_array(cluster_id, plugin, endpoint="", options=None, credentials=None, + lun_size=0, credential_ref=None, journal=JOURNAL_LOCAL, journal_lun_size=0, + journal_options=None, allow_local_kms_credentials=None) -> dict: + """Register (or re-register, e.g. to rotate credentials) the storage + array of a multi-attach cluster. + + The plugin is loaded and connected with the credentials and its + capabilities read before anything is saved. The Cluster record keeps the + plugin name, endpoint, options, LUN size and a credential reference + (``Cluster.array_credential_ref``), never a credential: + + * ``credentials`` (a flat dict of strings / SecretStr) are wrapped by the + cluster's credentials key in its KMS (Vault Transit or KMIP; the local + KMS only with ``allow_local_kms_credentials``), bound to the array id + and endpoint, and only the wrapped form is kept; + * ``credential_ref`` names a secret kept outside simplyblock instead + (``{"kind": "vault-kv", "mount", "path"[, "version"]}`` or + ``{"kind": "kmip-secret", "uid"[, "field"]}``): simplyblock reads it + when it connects and never stores it. + + ``journal`` places each node's journal: ``"local"`` on a disk or + partition of its host, ``"lun"`` on a LUN of its own that node adds + create (needs ``lun_size``) with ``journal_options`` updating + ``options`` (e.g. ``{"pool": "fast-pool"}``) and ``journal_lun_size`` + bytes (0 = 3% of ``lun_size``, at least 2 GiB). A change applies to + nodes added afterwards. + + ``allow_local_kms_credentials`` (True / False; None leaves it as it is) + sets ``Cluster.allow_local_kms_credentials``, the opt-in to keeping the + wrapped credentials in the local key store of a cluster without an + external KMS (test environments). It is saved only with a registration + that succeeds. + + Returns the array view (``array_view``) with the plugin's capabilities. + + Raises: + ValueError: not a multi-attach cluster, a bad journal placement, + both or a bad credential + source, the credentials cannot be stored or read, the plugin + cannot be loaded or connected, its capabilities do not fit the + cluster, or LUNs created through another plugin exist. + """ + from simplyblock_core import credentials as creds_mod + from simplyblock_core import kms + db = _db() + cluster = db.get_cluster_by_id(cluster_id) + if not multi_attach.is_multi_attach(cluster): + raise ValueError("an array is only used by a multi-attach cluster") + if not plugin: + raise ValueError("an array needs a plugin") + if int(lun_size or 0) < 0: + raise ValueError("lun_size must not be negative") + journal = (journal or JOURNAL_LOCAL).lower() + if journal not in JOURNAL_PLACEMENTS: + raise ValueError(f"journal must be one of {', '.join(JOURNAL_PLACEMENTS)}, not {journal!r}") + if int(journal_lun_size or 0) < 0: + raise ValueError("journal_lun_size must not be negative") + journal_opts = dict(journal_options or {}) + if journal == JOURNAL_LUN and not int(lun_size or 0): + raise ValueError("a journal LUN is created at node add next to the node's data LUN; " + "it needs lun_size") + if journal != JOURNAL_LUN and (journal_opts or int(journal_lun_size or 0)): + raise ValueError("journal_options and journal_lun_size apply to the journal placement 'lun' only") + creds = _wrap_credentials(credentials) + if creds and credential_ref: + raise ValueError("give the credentials or a reference to them, not both") + opts = dict(options or {}) + nodes = db.get_storage_nodes_by_cluster_id(cluster_id) + if (cluster.array_plugin and cluster.array_plugin != plugin + and _managed_devices(nodes)): + raise ValueError( + f"LUNs created through plugin {cluster.array_plugin!r} exist; the plugin cannot change") + array_id = cluster.array_id or str(uuid_mod.uuid4()) + context = array.credential_context(array_id, endpoint or "") + ref: dict = {} + if credential_ref: + try: + parsed = creds_mod.CREDENTIAL_REF_ADAPTER.validate_python(dict(credential_ref)) + except Exception as e: # noqa: BLE001 - pydantic + raise ValueError(f"credential_ref is not a valid reference: {e}") from None + if isinstance(parsed, creds_mod.ManagedCredentialRef): + raise ValueError("credential_ref names an external secret (vault-kv or kmip-secret); " + "give the credentials themselves to have them stored") + ref = parsed.model_dump() + try: + resolved = creds_mod.CredentialProvider(cluster, ttl_s=0).resolve(parsed, context) + except creds_mod.CredentialError as e: + raise ValueError(f"the referenced credentials could not be read: {e}") from None + creds = {name: resolved.get(name) for name in resolved.field_names + if name != creds_mod.CONTEXT_FIELD} + try: + inst = array.load_plugin(plugin, endpoint, opts, credentials=creds, cluster_id=cluster_id) + caps = dict(inst.capabilities() or {}) + except array.ArrayError as e: + raise ValueError(str(e)) from e + except Exception as e: # noqa: BLE001 - a vendor plugin + raise ValueError(f"array plugin {plugin!r} failed: {e}") from e + if journal_opts: + # The journal LUNs' options are checked by the plugin, as the + # array's are, before anything is saved. + try: + array.load_plugin(plugin, endpoint, dict(opts, **journal_opts), credentials=creds, + cluster_id=cluster_id) + except array.ArrayError as e: + raise ValueError(f"journal options: {e}") from e + except Exception as e: # noqa: BLE001 - a vendor plugin + raise ValueError(f"journal options: array plugin {plugin!r} failed: {e}") from e + max_att = int(caps.get("max_attachments") or 0) + # Owner, secondary, tertiary: FTT + 1 hosts per LUN. + needed = int(getattr(cluster, "max_fault_tolerance", 1) or 1) + 1 + if max_att and max_att < needed: + raise ValueError(f"the array presents a LUN to at most {max_att} hosts; the cluster needs {needed}") + + if allow_local_kms_credentials is not None: + # Asked for with the registration, so it applies to this very store; + # the record keeps it only if the registration is saved below. + cluster.allow_local_kms_credentials = bool(allow_local_kms_credentials) + if creds and not ref: + try: + ref = creds_mod.CredentialProvider(cluster).store(creds, context).model_dump() + except creds_mod.CredentialError as e: + raise ValueError(f"the array credentials could not be stored: {e}") from None + # A secret an earlier registration kept at the KMS path (before + # credential references) is removed once the new registration is saved. + legacy_secret = bool(cluster.array_has_credentials) and not cluster.array_credential_ref + + def mutate(fresh): + fresh.array_id = array_id + fresh.array_plugin = plugin + fresh.array_endpoint = endpoint or "" + fresh.array_options = opts + fresh.array_credential_ref = ref + fresh.array_has_credentials = bool(ref) + fresh.array_lun_size = int(lun_size or 0) + fresh.array_journal = journal + fresh.array_journal_lun_size = int(journal_lun_size or 0) + fresh.array_journal_options = journal_opts + if allow_local_kms_credentials is not None: + fresh.allow_local_kms_credentials = bool(allow_local_kms_credentials) + + db.atomic_update(cluster, mutate) + if legacy_secret: + try: + with kms.create_kms_connection(cluster) as conn: + conn.delete_secret(kms.array_secret_path(cluster_id, array_id)) + except Exception: # noqa: BLE001 + logger.exception("could not remove the array's earlier KMS secret") + array.forget_plugin(cluster_id) + cluster = db.get_cluster_by_id(cluster_id) + logger.info("Cluster %s: array %s registered (plugin %s, endpoint %s)", + cluster_id, array_id, plugin, endpoint or "-") + return array_view(cluster, caps) + + +def get_arrays(cluster_id) -> list[dict]: + """The registered arrays of a cluster (zero or one), without credentials.""" + cluster = _db().get_cluster_by_id(cluster_id) + if not cluster.array_plugin: + return [] + return [array_view(cluster)] + + +def delete_array(cluster_id, array_id, force=False) -> None: + """Remove the cluster's array registration and its credentials. + + Refused while LUNs created through it exist (a takeover could no longer + fence), unless ``force``. + + Raises: + KeyError: no array ``array_id``. + ValueError: LUNs of the array exist. + """ + from simplyblock_core import kms + db = _db() + cluster = db.get_cluster_by_id(cluster_id) + if not cluster.array_plugin or (cluster.array_id or "") != (array_id or ""): + raise KeyError(f"array {array_id} not found") + nodes = db.get_storage_nodes_by_cluster_id(cluster_id) + managed = [d.array_lun_id for _, d in _managed_devices(nodes)] + managed += [n.array_journal_lun_id for n in nodes if getattr(n, "array_journal_lun_id", "")] + if managed and not force: + raise ValueError(f"{len(managed)} LUN(s) created through the array exist; " + "takeovers would lose their fence") + # A credential reference needs no cleanup: a managed one exists only in + # the record, an external one belongs to whoever created it. + if cluster.array_id and cluster.array_has_credentials and not cluster.array_credential_ref: + with kms.create_kms_connection(cluster) as conn: + conn.delete_secret(kms.array_secret_path(cluster_id, cluster.array_id)) + + def mutate(fresh): + fresh.array_id = "" + fresh.array_plugin = "" + fresh.array_endpoint = "" + fresh.array_options = {} + fresh.array_credential_ref = {} + fresh.array_has_credentials = False + fresh.array_lun_size = 0 + fresh.array_journal = JOURNAL_LOCAL + fresh.array_journal_lun_size = 0 + fresh.array_journal_options = {} + + db.atomic_update(cluster, mutate) + logger.info("Cluster %s: array %s removed", cluster_id, array_id) + + +# ------------------------------------------------------------------- hosts + +#: SAN initiator keys the node agent reports (``san_initiators`` of its +#: /info; node_utils.get_san_initiators) and SAN plugins read. +SAN_INITIATOR_KEYS = ("iscsi_iqn", "fc_wwpns", "nvme_hostnqn", "sdc_guid") + + +def host_initiators(hostname="", cloud_instance=None, system_id="", ip="", + san_initiators=None) -> dict: + """What identifies a host on an array: its hostname, system id, IP, its + SAN initiators (iSCSI IQN, FC WWPNs, NVMe host NQN, PowerFlex SDC GUID, + as its agent reports them) and, on AWS, the EC2 instance id (reported by + the node agent from the instance metadata service when its cloud probe + runs).""" + out = {"hostname": hostname} + for key in SAN_INITIATOR_KEYS: + value = (san_initiators or {}).get(key) + if value: + out[key] = value + if ip: + out["ip"] = ip + if system_id: + out["system_id"] = system_id + cloud = cloud_instance or {} + if cloud.get("cloud") == "amazon" and str(cloud.get("id", "")).startswith("i-"): + out["ec2_instance_id"] = cloud["id"] + return out + + +def _agent_san_initiators(node) -> dict: + """The SAN initiators the node's agent reports now ({} when it cannot be + asked, or is older than the report).""" + try: + info, _ = node.client(timeout=15, retry=1).info() + except Exception as e: # noqa: BLE001 + logger.warning("Node %s: SAN initiators not read from its agent: %s", node.get_id(), e) + return {} + return dict((info or {}).get("san_initiators") or {}) + + +def _node_initiators(node) -> dict: + cloud = {"cloud": getattr(node, "cloud_name", ""), "id": getattr(node, "cloud_instance_id", "")} + return host_initiators(multi_attach.array_host(node), cloud, getattr(node, "system_uuid", ""), + getattr(node, "mgmt_ip", "") or "", _agent_san_initiators(node)) + + +def ensure_node_host(plugin, node, db=None) -> str: + """The node's host id on the array, registered and recorded on the node + (``StorageNode.array_host_id``) the first time.""" + host_id = getattr(node, "array_host_id", "") or "" + if host_id: + return host_id + host_id = plugin.ensure_host(multi_attach.array_host(node), _node_initiators(node)) + if not host_id: + raise ArrayLifecycleError(f"the array returned no host id for node {node.get_id()}") + db = db or _db() + + def _set(n): + n.array_host_id = host_id + db.atomic_update(node, _set) + node.array_host_id = host_id + return host_id + + +# ---------------------------------------------------------------- node add + +def lun_name(hostname, socket, journal=False) -> str: + return f"simplyblock-{hostname}-s{socket if socket is not None else 0}" + ("-journal" if journal else "") + + +def _find_in_inventory(inventory, wwn): + """The host's device of LUN ``wwn``: its multipath map where the LUN is + reached over several paths (FC, iSCSI with several sessions), else its + disk.""" + want = multi_attach.normalize_wwn(wwn) + matches = [d for d in inventory or [] + if d.get("type", "disk") in ("disk", "mpath") + and want in (multi_attach.normalize_wwn(d.get("wwn", "")), + multi_attach.normalize_wwn(d.get("serial", "")))] + maps = [d for d in matches if d.get("type") == "mpath"] + if maps: + return maps[0] + # A path of a map whose map is not listed (an older agent): never alone. + singles = [d for d in matches if not d.get("multipath_map")] + return singles[0] if singles else None + + +def provision_node_lun(cluster, snode_api, node_info, node_config, lblk_configured, + timeout=LUN_APPEAR_TIMEOUT_SEC, node_ip="", journal=False): + """Create (or find) the LUN of the node being added, present it to the + node's host and wait until the host sees it. + + With ``journal`` it is the node's journal LUN (``journal_on_lun``): + created with ``Cluster.array_journal_options`` updating the array's + options and ``journal_lun_size`` bytes, and its entry flagged as the + journal. It is presented to this node only, ever. + + Returns ``(entry, host_id)``: ``entry`` is an lblk config entry for the + LUN (identified by its serial; carries ``lun_wwn`` and ``array_lun_id``) + to add to the node's ``lblk_devices``, ``host_id`` the host's id on the + array. A configured entry of the same kind (data or journal) that already + carries an ``array_lun_id`` (a retried add) is reused. + + Raises: + ArrayLifecycleError: any step failed. + """ + hostname = node_info.get("hostname", "") + socket = node_config.get("socket") + try: + plugin = array.plugin_for_cluster( + cluster, dict(getattr(cluster, "array_journal_options", None) or {}) if journal else None) + host_id = plugin.ensure_host(hostname, host_initiators( + hostname, node_info.get("cloud_instance"), node_info.get("system_id", ""), node_ip, + node_info.get("san_initiators"))) + existing = next((e for e in lblk_configured or [] + if e.get("array_lun_id") and bool(e.get("journal")) == bool(journal)), None) + if existing: + lun = {"id": existing["array_lun_id"], "wwn": existing.get("lun_wwn", "")} + else: + tags = {"simplyblock-host": hostname, + "simplyblock-socket": str(socket if socket is not None else 0)} + if journal: + tags["simplyblock-role"] = "journal" + lun = plugin.create_lun(lun_name(hostname, socket, journal), + journal_lun_size(cluster) if journal else int(cluster.array_lun_size), + tags) + plugin.attach(lun["id"], host_id) + except array.ArrayError as e: + raise ArrayLifecycleError(f"array: {e}") from e + except Exception as e: # noqa: BLE001 - a vendor plugin + raise ArrayLifecycleError(f"array plugin failed: {e}") from e + logger.info("Node %s socket %s: %sLUN %s (%s) attached as host %s", + hostname, socket, "journal " if journal else "", lun["id"], lun["wwn"], host_id) + + deadline = time.monotonic() + timeout + while True: + _scan_new_luns(snode_api) + inventory, err = snode_api.get_blockdevices() + found = _find_in_inventory(inventory, lun["wwn"]) + if found: + break + if time.monotonic() >= deadline: + raise ArrayLifecycleError( + f"LUN {lun['id']} ({lun['wwn']}) did not appear on {hostname} within {timeout}s" + + (f": {err}" if err else "")) + _sleep(POLL_SEC) + entry = { + "name": found["name"], + "serial": found.get("serial") or lun["wwn"], + "size": int(found.get("size") or 0), + "by_id": found.get("by_id_path", ""), + "numa": int(found.get("numa_node", -1)), + "model": found.get("model", ""), + "type": "disk", + # A multipath map's kernel name (dm-N) makes it a bdev_uring device. + "kname": found.get("kname", found["name"]), + "lun_wwn": lun["wwn"], + "array_lun_id": lun["id"], + } + if journal: + entry["journal"] = True + return entry, host_id + + +def journal_lun_id(lblk_entries) -> str: + """The array id of the journal LUN among a node's lblk entries ("").""" + return next((e["array_lun_id"] for e in lblk_entries or [] + if e.get("journal") and e.get("array_lun_id")), "") + + +def add_lun_to_config(lblk_configured, entry) -> list: + """``lblk_configured`` with the LUN entry in place of an older entry for + the same LUN (same serial or array id). + + On a multi-attach cluster a node's local devices can only hold its + journal (a data device must be reachable from the standbys). When the + host config names exactly one local device and flags none as the + journal, that device is flagged, so the journal is never placed on the + LUN by the smallest-device fallback.""" + out = [dict(e) for e in lblk_configured or [] + if e.get("serial") != entry["serial"] + and not (e.get("array_lun_id") and e.get("array_lun_id") == entry["array_lun_id"])] + local = [e for e in out if not e.get("array_lun_id")] + if len(local) == 1 and not any(e.get("journal") for e in out): + local[0]["journal"] = True + out.append(entry) + return out + + +# ------------------------------------------------------------- attachments + +def _scan_new_luns(snode_api) -> None: + """Have the host look for a LUN just presented to it. A SAN host sees a + new LUN only after a SCSI / NVMe rescan (an EBS attachment shows up by + itself, so a failure, or an agent without the endpoint, is not fatal: + the caller keeps polling).""" + try: + _, err = snode_api.scan_new_luns() + if err: + logger.debug("scan_new_luns: %s", err) + except Exception as e: # noqa: BLE001 + logger.debug("scan_new_luns: %s", e) + + +def _lun_path_on(node, dev, timeout): + """The path of the device's LUN on ``node`` from its agent, or "".""" + owner_path = (dev.node_paths or {}).get(dev.node_id, {}).get("path", "") + deadline = time.monotonic() + timeout + while True: + try: + _scan_new_luns(node.client(timeout=15, retry=1)) + entries, _ = node.client(timeout=15, retry=1).lun_info(dev.lun_wwn) + except Exception as e: # noqa: BLE001 + logger.warning("lun_info on %s: %s", node.get_id(), e) + entries = None + for entry in entries or []: + if owner_path and (owner_path in (entry.get("by_id_paths") or []) or owner_path == entry.get("by_id_path")): + return owner_path + if entries: + first = entries[0] + return first.get("by_id_path") or first.get("device_path") or "" + if time.monotonic() >= deadline: + return "" + _sleep(POLL_SEC) + + +def desired_hosts(owner, dev, nodes_by_id) -> set: + """Node ids the LUN of ``dev`` must be presented to: its owner, the + owner's LVS secondary and tertiary, and the node serving it -- minus + removed nodes and nodes fenced from it (the serving node always stays).""" + serving = dev.serving_node() + want = {owner.get_id(), getattr(owner, "secondary_node_id", ""), + getattr(owner, "tertiary_node_id", ""), serving} + fenced = set(dev.fenced_node_ids or []) + out = set() + for nid in want: + node = nodes_by_id.get(nid) + if not nid or node is None or node.status == _STATUS_REMOVED: + continue + if nid in fenced and nid != serving: + continue + out.add(nid) + return out + + +def reconcile_lun_attachments(cluster_id, path_timeout=LUN_PATH_TIMEOUT_SEC) -> dict: + """Present every array-created LUN of the cluster to the nodes in + ``desired_hosts`` and only to them, and record its path on each. + + Attaches before it detaches, never detaches the serving node, never + attaches a fenced node. A node already recorded in + ``array_attached_node_ids`` costs no array call, so the periodic run is + free while nothing changes. Returns ``{"attached": [...], "detached": + [...], "errors": {device_id: msg}}``; never raises for one LUN.""" + report: dict = {"attached": [], "detached": [], "errors": {}} + db = _db() + cluster = db.get_cluster_by_id(cluster_id) + if not array_configured(cluster): + return report + nodes = db.get_storage_nodes_by_cluster_id(cluster_id) + nodes_by_id = {n.get_id(): n for n in nodes} + work = [] + for owner, dev in _managed_devices(nodes): + if owner.status == _STATUS_REMOVED: + continue + want = desired_hosts(owner, dev, nodes_by_id) + have = set(dev.array_attached_node_ids or []) + missing_paths = [nid for nid in want + if nid != owner.get_id() and nid not in (dev.node_paths or {})] + if want != have or missing_paths: + work.append((owner, dev, want)) + if not work: + return report + try: + plugin = array.plugin_for_cluster(cluster) + except array.ArrayError as e: + for _, dev, _ in work: + report["errors"][dev.get_id()] = str(e) + logger.error("LUN attachments of cluster %s not reconciled: %s", cluster_id, e) + return report + + for owner, dev, want in work: + try: + _reconcile_one(db, plugin, owner, dev, want, nodes_by_id, path_timeout, report) + except Exception as e: # noqa: BLE001 - one LUN never stops the rest + report["errors"][dev.get_id()] = str(e) + logger.error("LUN %s of device %s: attachments not reconciled: %s", + dev.array_lun_id, dev.get_id(), e) + if report["errors"]: + _event(cluster_id, f"Array LUN attachments incomplete: {report['errors']}", error=True) + return report + + +def _reconcile_one(db, plugin, owner, dev, want, nodes_by_id, path_timeout, report): + lun_id = dev.array_lun_id + serving = dev.serving_node() + host_of = {} + for nid in want: + host_of[nid] = ensure_node_host(plugin, nodes_by_id[nid], db) + on_array = set(plugin.get(lun_id).get("hosts") or []) + node_of_host = {} + for n in nodes_by_id.values(): + hid = getattr(n, "array_host_id", "") or "" + if hid: + node_of_host[hid] = n.get_id() + for nid in sorted(want): + if host_of[nid] not in on_array: + plugin.attach(lun_id, host_of[nid]) + report["attached"].append((dev.get_id(), nid)) + logger.info("LUN %s of device %s attached to node %s", lun_id, dev.get_id(), nid) + for host_id in sorted(on_array): + nid = node_of_host.get(host_id) + if nid is None: + # A host no node of the cluster is registered as: left alone + # (release_node_luns detaches a deleted node's host). + logger.warning("LUN %s of device %s is presented to unknown host %s; left alone", + lun_id, dev.get_id(), host_id) + continue + if nid in want or nid == serving: + continue + plugin.detach(lun_id, host_id) + report["detached"].append((dev.get_id(), nid)) + logger.info("LUN %s of device %s detached from node %s", lun_id, dev.get_id(), nid) + + paths = {} + for nid in want: + if nid == owner.get_id() or nid in (dev.node_paths or {}): + continue + path = _lun_path_on(nodes_by_id[nid], dev, path_timeout) + if path: + paths[nid] = {"path": path, "bdev": ""} + else: + report["errors"][dev.get_id()] = f"LUN not visible on node {nid} yet" + attached = sorted(want) + device_id = dev.get_id() + + def _mut(n): + for d in n.nvme_devices or []: + if d.get_id() != device_id: + continue + d.array_attached_node_ids = attached + np = dict(d.node_paths or {}) + # A node that lost the role loses its path; the serving node's + # entry (written by the takeover) is never dropped here. + for nid in list(np): + if nid not in want and nid != d.serving_node() and nid != n.get_id(): + np.pop(nid) + for nid, p in paths.items(): + np.setdefault(nid, p) + d.node_paths = np + return True + return False + db.atomic_update(owner, _mut) + + +_reconcile_threads: dict = {} +_reconcile_lock = threading.Lock() + + +def reconcile_in_background(cluster) -> bool: + """Start ``reconcile_lun_attachments`` for the cluster in a daemon + thread unless one is running (the periodic hook of the health-check + service). Returns whether one was started.""" + if not array_configured(cluster): + return False + cid = cluster.get_id() + with _reconcile_lock: + t = _reconcile_threads.get(cid) + if t is not None and t.is_alive(): + return False + t = threading.Thread(target=_safe_reconcile, args=(cid,), daemon=True) + _reconcile_threads[cid] = t + t.start() + return True + + +def _safe_reconcile(cluster_id): + try: + from simplyblock_core.models.lock import DbLock + with DbLock(f"array_luns/{cluster_id}", timeout=0): + reconcile_lun_attachments(cluster_id) + except Exception as e: # noqa: BLE001 - a background pass + logger.warning("LUN attachment reconcile of cluster %s: %s", cluster_id, e) + + +def reconcile_after_role_change(cluster_id) -> dict: + """The hook after LVS roles were assigned (activation, expansion): one + reconcile pass, synchronous, under the cluster's array lock. Never + raises; problems are in the report and an event.""" + try: + cluster = _db().get_cluster_by_id(cluster_id) + if not array_configured(cluster): + return {"attached": [], "detached": [], "errors": {}} + from simplyblock_core.models.lock import DbLock + with DbLock(f"array_luns/{cluster_id}"): + return reconcile_lun_attachments(cluster_id) + except Exception as e: # noqa: BLE001 + logger.error("LUN attachments of cluster %s not reconciled: %s", cluster_id, e) + return {"attached": [], "detached": [], "errors": {"*": str(e)}} + + +# ------------------------------------------------------------------ growth + +def grow_luns(cluster_id, factor=None, visible_timeout=LUN_GROW_VISIBLE_TIMEOUT_SEC, size=None) -> dict: + """Grow every array-created data LUN of the cluster at the array, then + wait until every ONLINE node the LUN is presented to sees the new size + (agent rescan; dm-multipath maps are resized too). + + Either by ``factor`` (> 1, relative to each LUN's size now), or to + ``size`` bytes: a LUN already that large is left alone, so a growth to a + size can be repeated after a failure without growing any LUN twice, and + the cluster's LUN size for nodes added later (``Cluster.array_lun_size``) + becomes ``size`` once every LUN has it. + + Returns ``{"luns": {device_id: {"lun_id", "old_size", "new_size"}}, + "errors": {device_id: msg}}``. The device growth itself (alceml, weights) + is the regular grow-devices path that follows. + + Raises: + ValueError: not an array cluster, a bad factor or size, or both or + neither given. + """ + if (factor is None) == (size is None): + raise ValueError("give a growth factor or a LUN size, not both") + if size is not None: + try: + size = int(size) + except (TypeError, ValueError) as e: + raise ValueError("the LUN size must be a number of bytes") from e + if size <= 0: + raise ValueError("the LUN size must be positive") + else: + try: + factor = float(factor) + except (TypeError, ValueError) as e: + raise ValueError("the growth factor must be a number") from e + if not math.isfinite(factor) or factor <= 1.0: + raise ValueError("the growth factor must be greater than 1") + db = _db() + cluster = db.get_cluster_by_id(cluster_id) + if not array_configured(cluster): + raise ValueError("LUN growth through the array needs a multi-attach cluster with an array") + try: + plugin = array.plugin_for_cluster(cluster) + except array.ArrayError as e: + raise ValueError(str(e)) from e + nodes = db.get_storage_nodes_by_cluster_id(cluster_id) + nodes_by_id = {n.get_id(): n for n in nodes} + out: dict = {"luns": {}, "errors": {}} + for owner, dev in _managed_devices(nodes): + if owner.status == _STATUS_REMOVED or dev.status in ("removed", "failed", "failed_and_migrated"): + continue + did = dev.get_id() + try: + old = int(plugin.get(dev.array_lun_id).get("size") or 0) or int(dev.size) + target = max(old, size) if size is not None else math.ceil(old * factor) + if target > old: + plugin.resize(dev.array_lun_id, target) + new = int(plugin.get(dev.array_lun_id).get("size") or target) + out["luns"][did] = {"lun_id": dev.array_lun_id, "old_size": old, "new_size": new} + except Exception as e: # noqa: BLE001 + out["errors"][did] = f"array resize of LUN {dev.array_lun_id} failed: {e}" + continue + err = _wait_visible(dev, nodes_by_id, new, visible_timeout) + if err: + out["errors"][did] = err + if size is not None and not out["errors"] and size > int(cluster.array_lun_size or 0): + def _set(fresh): + fresh.array_lun_size = max(int(fresh.array_lun_size or 0), size) + db.atomic_update(cluster, _set) + return out + + +def _wait_visible(dev, nodes_by_id, size, timeout) -> str: + """Rescan the LUN on every ONLINE node it has a path on until each sees + ``size`` bytes (EBS rounds to GiB, so at least ``size``). Returns "" or + what is still missing.""" + pending = {nid: p.get("path") for nid, p in (dev.node_paths or {}).items() + if p.get("path") and nid in nodes_by_id + and nodes_by_id[nid].status == _STATUS_ONLINE} + deadline = time.monotonic() + timeout + last = {} + while pending: + for nid, path in list(pending.items()): + try: + res, err = nodes_by_id[nid].client(timeout=60, retry=1).rescan_block_device(path) + except Exception as e: # noqa: BLE001 + res, err = None, str(e) + seen = int((res or {}).get("size") or 0) + last[nid] = err or f"sees {seen} bytes" + if seen >= size: + pending.pop(nid) + if not pending: + break + if time.monotonic() >= deadline: + return "new size not visible on " + ", ".join(f"{n} ({last.get(n)})" for n in sorted(pending)) + _sleep(POLL_SEC) + return "" + + +# -------------------------------------------------------- removal / delete + +def release_node_luns(node) -> None: + """A storage node is being deleted: detach its own array-created LUNs + from every host, and detach other nodes' LUNs from its host unless it + serves them. Nothing is deleted (only a cluster delete deletes LUNs). + Failures are logged and reported as an event, never raised.""" + try: + db = _db() + cluster = db.get_cluster_by_id(node.cluster_id) + if not array_configured(cluster): + return + nodes = db.get_storage_nodes_by_cluster_id(node.cluster_id) + work: list[tuple] = [] + node_host = getattr(node, "array_host_id", "") or "" + for owner, dev in _managed_devices(nodes): + if owner.get_id() == node.get_id(): + work.append((dev.array_lun_id, None)) + elif node_host and dev.serving_node() != node.get_id(): + work.append((dev.array_lun_id, node_host)) + if getattr(node, "array_journal_lun_id", ""): + work.append((node.array_journal_lun_id, None)) + if not work: + return + plugin = array.plugin_for_cluster(cluster) + except Exception as e: # noqa: BLE001 + logger.error("LUNs of node %s not released: %s", node.get_id(), e) + _event(node.cluster_id, f"LUNs of node {node.get_id()} not released at the array: {e}", error=True) + return + errors = [] + for lun_id, host in work: + try: + hosts = [host] if host else list(plugin.get(lun_id).get("hosts") or []) + for h in hosts: + # The node is gone; its host may not cooperate. + plugin.detach(lun_id, h, force=True) + except Exception as e: # noqa: BLE001 + errors.append(f"{lun_id}: {e}") + if errors: + logger.error("LUNs of node %s not fully released: %s", node.get_id(), errors) + _event(node.cluster_id, f"LUNs of node {node.get_id()} not fully released: {errors}", error=True) + + +def delete_cluster_luns(cluster, force=False) -> None: + """A cluster is being deleted: delete every LUN its plugin created + (detached from every host first) and the array's KMS secret. + + Raises: + ValueError: a LUN could not be deleted (unless ``force``, which logs + and goes on: the LUNs left are listed in the log). + """ + if not array_configured(cluster): + return + errors = [] + # The LUNs recorded on the cluster's devices, plus whatever the plugin + # lists (SAN plugins cannot list: their recorded LUNs are deleted). + nodes = _db().get_storage_nodes_by_cluster_id(cluster.get_id()) + recorded = [d.array_lun_id for _, d in _managed_devices(nodes)] + recorded += [n.array_journal_lun_id for n in nodes if getattr(n, "array_journal_lun_id", "")] + luns: list = [] + try: + plugin = array.plugin_for_cluster(cluster) + except Exception as e: # noqa: BLE001 + plugin = None + errors.append(f"array: {e}") + if plugin is not None: + try: + luns = list(plugin.list_luns()) + except NotImplementedError: + logger.info("Cluster %s: the array plugin cannot list LUNs; deleting the %d recorded", + cluster.get_id(), len(recorded)) + except Exception as e: # noqa: BLE001 + errors.append(f"array: {e}") + listed = {lun["id"] for lun in luns} + luns += [{"id": lun_id} for lun_id in dict.fromkeys(recorded) if lun_id not in listed] + for lun in luns: + try: + plugin.delete_lun(lun["id"]) # type: ignore[union-attr] + logger.info("Cluster %s: LUN %s deleted", cluster.get_id(), lun["id"]) + except Exception as e: # noqa: BLE001 + errors.append(f"{lun['id']}: {e}") + if errors and not force: + raise ValueError(f"array LUNs of the cluster could not be deleted: {'; '.join(errors)}") + if errors: + logger.error("Cluster %s deleted with array LUNs left behind: %s", cluster.get_id(), errors) + if cluster.array_id and cluster.array_has_credentials and not cluster.array_credential_ref: + from simplyblock_core import kms + try: + with kms.create_kms_connection(cluster) as conn: + conn.delete_secret(kms.array_secret_path(cluster.get_id(), cluster.array_id)) + except Exception as e: # noqa: BLE001 + if not force: + raise ValueError(f"the array credentials could not be deleted from the KMS: {e}") from e + logger.error("Array credentials of cluster %s left in the KMS: %s", cluster.get_id(), e) diff --git a/simplyblock_core/controllers/cluster_events.py b/simplyblock_core/controllers/cluster_events.py index 6380f05c48..6206784da3 100644 --- a/simplyblock_core/controllers/cluster_events.py +++ b/simplyblock_core/controllers/cluster_events.py @@ -71,6 +71,34 @@ def _cluster_cap_event(cluster, msg, event_level): event_level=event_level) +def cluster_floor_violation(cluster, violations): + """The monitor found the online resources below a floor (the cluster is + DEGRADED for it); ``violations`` are the floor texts of + simplyblock_core.floors.floor_violations. One event per change of the set.""" + msg = "Cluster below resource floor: " + "; ".join(violations) + return ec.log_event_cluster( + cluster_id=cluster.get_id(), + node_id=cluster.get_id(), + domain=ec.DOMAIN_CLUSTER, + event=ec.EVENT_FLOOR, + db_object=cluster, + caused_by=ec.CAUSED_BY_MONITOR, + message=msg, + event_level=EventObj.LEVEL_WARN) + + +def cluster_floor_restored(cluster): + return ec.log_event_cluster( + cluster_id=cluster.get_id(), + node_id=cluster.get_id(), + domain=ec.DOMAIN_CLUSTER, + event=ec.EVENT_FLOOR, + db_object=cluster, + caused_by=ec.CAUSED_BY_MONITOR, + message="Cluster back at or above every resource floor", + event_level=EventObj.LEVEL_INFO) + + def cluster_cap_warn(cluster, util): msg = f"Cluster absolute capacity reached: {util}%" return _cluster_cap_event(cluster, msg, event_level=EventObj.LEVEL_WARN) diff --git a/simplyblock_core/controllers/cluster_expansion/planner.py b/simplyblock_core/controllers/cluster_expansion/planner.py index fa1f8d5b76..597dc84db5 100644 --- a/simplyblock_core/controllers/cluster_expansion/planner.py +++ b/simplyblock_core/controllers/cluster_expansion/planner.py @@ -503,11 +503,24 @@ def fd_balance_violation( return None +def fd_activation_min_domains(npcs: int, ndcs: int = 0) -> int: + """The failure domains a fresh activation needs: the rotation's spare + rule (npcs+2, see fd_activation_domain_count_violation) and the EC floor + of ndcs+npcs domains (simplyblock_core.floors, floor 3), whichever is + higher.""" + return max(npcs + 2, ndcs + npcs) + + def fd_activation_domain_count_violation( - npcs: int, distinct_domain_count: int, + npcs: int, distinct_domain_count: int, ndcs: int = 0, ) -> str | None: """Validate the number of distinct failure domains for fresh activation. + Two rules, the higher one binds (``fd_activation_min_domains``): the EC + floor of ``ndcs+npcs`` domains -- a stripe's chunks must each have a + domain of their own when failure domains are used at all -- and the + rotation-spare rule below. + A 2-FD layout can never absorb a second independent failure once one domain is fully down (confirmed with the backend team), so it is not supported at any npcs level. @@ -533,12 +546,13 @@ def fd_activation_domain_count_violation( human-readable reason on violation, ``None`` when the count is acceptable. """ - min_domains = npcs + 2 + min_domains = fd_activation_min_domains(npcs, ndcs) if distinct_domain_count < min_domains: + rule = ("n+k" if ndcs + npcs >= npcs + 2 else "k+2") return ( - f"failure domains are enabled with npcs={npcs}, which requires at " - f"least {min_domains} distinct failure domains (2 domains is not " - f"supported at any npcs level); currently have " + f"failure domains are enabled with ndcs={ndcs}, npcs={npcs}, which " + f"requires at least {min_domains} distinct failure domains ({rule}; " + f"2 domains is not supported at any npcs level); currently have " f"{distinct_domain_count}. Add hosts in additional domains, or " f"disable failure domains, then activate.") return None diff --git a/simplyblock_core/controllers/device_controller.py b/simplyblock_core/controllers/device_controller.py index 7be0a6c422..28b2044541 100644 --- a/simplyblock_core/controllers/device_controller.py +++ b/simplyblock_core/controllers/device_controller.py @@ -7,13 +7,13 @@ constants, distr_controller, jm_raid, + multi_attach, storage_node_ops, utils, ) from simplyblock_core.controllers import device_events, tasks_controller from simplyblock_core.db_controller import DBController from simplyblock_core.models.cluster import Cluster -from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.nvme_device import JMDevice, NVMeDevice from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.prom_client import PromClient @@ -129,6 +129,10 @@ def get_storage_node_by_jm_device(db_controller: DBController, id) -> StorageNod # when IO resumes (2026-07-06 failback incident: placement failures because # peers still saw the recovering node's devices UNAVAILABLE). CAUSE_NODE_RECOVERY = "node_recovery" +# Multi-attach array LUN mode: the device's LUN is served by another node +# (controllers/lun_takeover) while its owner is not ONLINE. The device may go +# ONLINE as long as its SERVING node is ONLINE. +CAUSE_LUN_TAKEOVER = "lun_takeover" def _atomic_device_set(db_controller, node_id, device_id, apply_fields): @@ -201,6 +205,22 @@ def _connect_peer(peer_id): th.join() +def _serving_node_online(db_controller, device): + try: + server = db_controller.get_storage_node_by_id(device.serving_node()) + except KeyError: + return False + return server.status == StorageNode.STATUS_ONLINE + + +def _served_by_online_node(db_controller, device, cause): + """CAUSE_LUN_TAKEOVER only: the multi-attach device is served by another + node (serving_node_id) and that node is ONLINE.""" + if cause != CAUSE_LUN_TAKEOVER or not multi_attach.served_elsewhere(device, device.node_id): + return False + return _serving_node_online(db_controller, device) + + def device_set_state(device_id, state, cause=CAUSE_OTHER, connect_peers=True): db_controller = DBController() try: @@ -237,13 +257,31 @@ def device_set_state(device_id, state, cause=CAUSE_OTHER, connect_peers=True): # ONLINE node, so it is unaffected. if (state == NVMeDevice.STATUS_ONLINE and snode.status != StorageNode.STATUS_ONLINE - and cause not in (CAUSE_DEVICE_RESTART, CAUSE_NODE_RECOVERY)): + and cause not in (CAUSE_DEVICE_RESTART, CAUSE_NODE_RECOVERY) + and not _served_by_online_node(db_controller, device, cause)): logger.warning( f"Refusing to set device {device_id} ONLINE while node " f"{snode.get_id()} is {snode.status} (cause={cause}); " f"node bring-up owns device re-online") return False + # Multi-attach: a device whose LUN another node serves is ONLINE only + # while that node is; its owner's recovery does not bring it back (the + # owner built nothing for it). + if (state == NVMeDevice.STATUS_ONLINE + and multi_attach.served_elsewhere(device, snode.get_id()) + and not _serving_node_online(db_controller, device)): + logger.warning( + f"Refusing to set device {device_id} ONLINE: its LUN is served by node " + f"{device.serving_node()}, which is not ONLINE (cause={cause})") + return False + if (state == NVMeDevice.STATUS_ONLINE + and multi_attach.fenced_from(device, device.serving_node())): + logger.warning( + f"Refusing to set device {device_id} ONLINE: node {device.serving_node()} " + f"is fenced from its LUN (cause={cause})") + return False + # Failed is a terminal state. The only allowed exits are: # STATUS_FAILED → STATUS_FAILED_AND_MIGRATED (failure data migration) # STATUS_FAILED → STATUS_ONLINE (explicit device restart) @@ -529,6 +567,11 @@ def device_repair(device_id, force=False): "Device %s was removed by operator request; not repairing", device_id) return False + if multi_attach.not_servable_here(device, device.node_id): + logger.info("Device %s: its LUN is served by node %s (owner fenced: %s); " + "the owner does not repair it", device_id, device.serving_node(), + multi_attach.fenced_from(device, device.node_id)) + return False if device.status != NVMeDevice.STATUS_ONLINE and device.retries_exhausted and not force: logger.debug("Device %s repair budget exhausted; not retrying", device_id) @@ -678,6 +721,11 @@ def device_repair_due(device): # The operator removed this device on purpose. Never resurrect it -- # not on any attempt, and not under force. return False + if multi_attach.not_servable_here(device, device.node_id): + # Multi-attach: another node serves the LUN (or the owner is fenced + # from it). Repair rebuilds the stack on the owner, which must not + # hold one; the serving node's stack is the takeover's business. + return False schedule = constants.DEVICE_REPAIR_BACKOFF_SEC if device.repair_attempts >= len(schedule): return False @@ -756,6 +804,7 @@ def _def_create_device_stack(device_obj, snode, force=False, clear_data=False): checksum_method=checksum_method, cache_size=cache_size, cache_eviction_threshold=cache_eviction_threshold, + **multi_attach.alceml_create_kwargs(device_obj, snode, cluster), ) except RPCException as e: logger.error("Failed to create alceml bdev %s: %s", alceml_name, e) @@ -776,7 +825,8 @@ def _def_create_device_stack(device_obj, snode, force=False, clear_data=False): else: logger.info(f"bdev already exists {pt_name}") - subsystem_nqn = snode.subsystem + ":dev:" + alceml_id + # Multi-attach: host-independent NQN (multi_attach.device_subsystem_nqn). + subsystem_nqn = multi_attach.device_subsystem_nqn(snode, device_obj, cluster) namespace_found = False subsys_found = False ret = rpc_client.subsystem_get(subsystem_nqn) @@ -790,7 +840,9 @@ def _def_create_device_stack(device_obj, snode, force=False, clear_data=False): if not subsys_found: logger.info("Creating subsystem %s", subsystem_nqn) - ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', alceml_id) + ret = rpc_client.subsystem_create( + subsystem_nqn, 'sbcli-cn', alceml_id, + **multi_attach.subsystem_create_kwargs(snode, device_obj, db_controller)) for iface in snode.data_nics: if iface.ip4_address: ret = rpc_client.listeners_create(subsystem_nqn, iface.trtype, iface.listen_address, snode.nvmf_port) @@ -801,7 +853,14 @@ def _def_create_device_stack(device_obj, snode, force=False, clear_data=False): if not namespace_found: logger.info(f"Adding {pt_name} to the subsystem") - ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name) + if multi_attach.is_multi_attach(device_obj): + # Every node that exports the LUN must use the same namespace + # UUID: SPDK multipath joins paths by namespace identity. + ret = rpc_client.nvmf_subsystem_add_ns( + subsystem_nqn, pt_name, multi_attach.namespace_uuid(device_obj), + nguid=multi_attach.namespace_nguid(device_obj)) + else: + ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name) else: logger.info(f"bdev already added to subsys {pt_name}") @@ -902,10 +961,12 @@ def restart_device(device_id, force=False): if not snode.rpc_client().bdev_get(device_obj.nvme_bdev): try: filename = device_obj.by_id_path or device_obj.device_path + live = None try: inventory, _ = snode.client(timeout=30, retry=1).get_blockdevices() for blk in inventory or []: if blk.get("serial") == device_obj.serial_number: + live = blk filename = blk.get("by_id_path") or blk.get("device_path") device_obj.device_path = blk.get("device_path", device_obj.device_path) device_obj.by_id_path = blk.get("by_id_path", device_obj.by_id_path) @@ -915,7 +976,16 @@ def restart_device(device_id, force=False): if not filename: logger.error(f"No block device path known for {device_id}") return False - snode.rpc_client().bdev_aio_create(device_obj.nvme_bdev, filename) + # The block size the device was formatted with (0: legacy). + block_size = (device_obj.aio_block_size + or utils.recorded_aio_block_size(snode, device_obj.serial_number) + or 0) + # aio, or uring for a device-mapper node. + device_obj.lblk_io_engine = utils.lblk_io_engine_of_device( + snode, device_obj, live, filename) + utils.create_aio_bdev(snode.rpc_client(), device_obj.nvme_bdev, + filename, block_size, + io_engine=device_obj.lblk_io_engine) snode.rpc_client().bdev_examine(device_obj.nvme_bdev) snode.rpc_client().bdev_wait_for_examine() snode.rpc_client().bdev_set_qd_sampling_period( @@ -1834,10 +1904,12 @@ def new_device_from_failed(device_id): if not device_node.rpc_client().bdev_get(device.nvme_bdev): try: filename = device.by_id_path or device.device_path + live = None try: inventory, _ = device_node.client(timeout=30, retry=1).get_blockdevices() for blk in inventory or []: if blk.get("serial") == device.serial_number: + live = blk filename = blk.get("by_id_path") or blk.get("device_path") break except Exception as e: @@ -1845,7 +1917,17 @@ def new_device_from_failed(device_id): if not filename: logger.error(f"No block device path known for {device_id}") return False - device_node.rpc_client().bdev_aio_create(device.nvme_bdev, filename) + # The block size the device was formatted with (0: legacy). + block_size = (device.aio_block_size + or utils.recorded_aio_block_size(device_node, device.serial_number) + or 0) + # aio, or uring for a device-mapper node; the new device + # record below copies it. + device.lblk_io_engine = utils.lblk_io_engine_of_device( + device_node, device, live, filename) + utils.create_aio_bdev(device_node.rpc_client(), device.nvme_bdev, + filename, block_size, + io_engine=device.lblk_io_engine) device_node.rpc_client().bdev_examine(device.nvme_bdev) device_node.rpc_client().bdev_wait_for_examine() device_node.rpc_client().bdev_set_qd_sampling_period( @@ -2101,35 +2183,34 @@ def _apply(dev): # --------------------------------------------------------------------------- -# Online growth of lblk devices (a LUN grown on its storage array). +# Online growth of lblk devices (LUNs grown on their storage array). # # The base bdev of an lblk device is an SPDK AIO bdev over a kernel block # device, and alceml sits directly on it. alceml can swap its backing bdev # for a larger one without losing state: resize_begin holds new I/O, drains # pending I/O and closes the backing bdev; resize_end reopens it and takes -# its new size, so the pages beyond the old end become allocatable. Layers -# above alceml see a fixed virtual size and need no change. +# the size of the whole block device (alceml has no size limit of its own), +# so the pages beyond the old end become allocatable. Layers above alceml see +# a fixed virtual size and need no change. # -# Growing a device does not change its cluster-map weight. The weight is -# derived from NVMeDevice.placement_size, the size the device had when it was -# placed, and distrib's map spec (message_format_rpcs__distrib__v5.txt, -# weight_int) asks for weights never to change, because a change re-places -# 50% or more of the data. Weights only matter relative to each other, so when -# every data device grows by the same factor (the normal case: equal LUNs grown -# together on the array), nothing has to change and nothing moves. +# Growth is uniform only. The cluster-map weight of a device is derived from +# NVMeDevice.placement_size, the size the device had when it was placed, and +# never changes. Weights only matter relative to each other, so the cluster +# can grow without any data movement exactly when every data device grows by +# the same factor. grow_cluster_devices therefore rescans every data device, +# takes the SMALLEST growth factor across all of them, and grows every device +# by exactly that factor: with three LUNs grown by 1.5 and three by 2.0, +# every device grows by 1.5. A device whose block device did not grow keeps +# the factor at 1.0, and nothing grows. The rest of a larger LUN stays unused: +# placement fills every device in proportion to its weight, so it is never +# reached, and the capacity statistics leave it out (NVMeDevice.backend_size). # -# After an uneven growth the frozen weights no longer match the capacities, -# and the added capacity is not fully usable. grow_cluster_devices then sets -# every weight to the device's current size, pushes the new map to all nodes, -# and starts the cluster rebalance (rebalance_after_growth). That re-places a -# large part of the data, in the background and within the migration share of -# the QoS budget. Correctness does not depend on the static placement: distrib -# finds data by its recorded location, including locations that were correct -# under an earlier map (alg_remapped_ranges.hpp, get_location_latest, "STALE"), -# which is the same mechanism a device addition relies on. +# A block device smaller than its storage device is never accepted: the +# rescan records it, logs an error event, raises the device_shrunk alert, and +# nothing grows until the LUN is restored and rescanned. # -# A device that enters the cluster later gets a weight on the same scale as the -# grown devices (storage_node_ops.scaled_placement_size), so a uniform growth +# A device that enters the cluster later gets a weight on the same scale as +# the grown devices (storage_node_ops.scaled_placement_size), so a growth # followed by an expansion does not overfill the new device. # --------------------------------------------------------------------------- @@ -2138,6 +2219,10 @@ def _apply(dev): RESIZE_RPC_TIMEOUT_SEC = 120 #: Attempts to reopen the backing bdev before the device is given up. RESIZE_END_ATTEMPTS = 3 +#: A cluster-wide growth factor below this counts as no growth. +GROWTH_MIN_FACTOR = 1.001 +#: Grown device sizes are rounded down to this alignment. +GROWTH_SIZE_ALIGN = 1024 * 1024 def cluster_growth_uniformity(cluster_id): @@ -2147,9 +2232,56 @@ def cluster_growth_uniformity(cluster_id): return storage_node_ops.cluster_growth_ratios(DBController(), cluster_id) -def _check_resizable(db_controller, dev, snode): - """Raise ValueError when ``dev`` cannot be grown online right now.""" - cluster = db_controller.get_cluster_by_id(snode.cluster_id) +def _growth_devices(db_controller, cluster_id): + """``(node, device)`` for every data device of the cluster that counts + for the growth factor: the same set as the cluster-map weights.""" + pairs = [] + for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id): + if node.is_secondary_node: + continue + for dev in node.nvme_devices: + if dev.status in (NVMeDevice.STATUS_JM, NVMeDevice.STATUS_NEW, + NVMeDevice.STATUS_FAILED, NVMeDevice.STATUS_FAILED_AND_MIGRATED): + continue + if dev.cluster_device_order < 0: + continue + pairs.append((node, dev)) + return pairs + + +def _resizable_errors(db_controller, cluster, pairs) -> dict: + """``{device_id: reason}`` for every device of ``pairs`` that cannot be + grown online right now (_check_resizable on its serving node).""" + errors = {} + for node, dev in pairs: + try: + host = _growth_host(db_controller, node, dev)[0] + _check_resizable(cluster, dev, node, host) + except (ValueError, KeyError) as e: + errors[dev.get_id()] = str(e) + return errors + + +def _growth_host(db_controller, snode, dev): + """``(node, base_bdev, path)`` where the alceml of ``dev`` runs now: the + owner ``snode``, or for a multi-attach device whose LUN another node took + over (lun_takeover), that serving node, with the base bdev and block + device path the takeover recorded for it.""" + if not multi_attach.served_elsewhere(dev, snode.get_id()): + return snode, dev.nvme_bdev, dev.by_id_path or dev.device_path + host = db_controller.get_storage_node_by_id(dev.serving_node()) + p = (dev.node_paths or {}).get(host.get_id()) or {} + base = p.get("bdev") or utils.aio_bdev_name_for_serial(dev.serial_number) + return host, base, p.get("path") or dev.by_id_path or dev.device_path + + +def _check_resizable(cluster, dev, snode, host=None): + """Raise ValueError when ``dev`` (owned by ``snode``) cannot be grown + online right now. ``host`` is the node whose alceml serves it + (_growth_host), the owner by default: a LUN served elsewhere after a + takeover is grown on its serving node. The owner must still be online, + so growth never runs while a node of the cluster is out.""" + host = host or snode if cluster.device_mode != constants.DEVICE_MODE_LBLK or dev.bdev_type != "aio": raise ValueError("Online resizing is only supported for Linux block devices (lblk device mode)") if dev.status != NVMeDevice.STATUS_ONLINE: @@ -2162,10 +2294,28 @@ def _check_resizable(db_controller, dev, snode): raise ValueError("Device runs with a testing or hang bdev under alceml, remove it first") if not dev.alceml_bdev or not dev.nvme_bdev: raise ValueError("Device has no alceml or AIO bdev recorded") + if host.get_id() != snode.get_id() and host.status != StorageNode.STATUS_ONLINE: + raise ValueError(f"The device's LUN is served by node {host.get_id()} after a takeover; " + f"that node must be online, current status: {host.status}") + if multi_attach.served_elsewhere(dev, host.get_id()): + raise ValueError(f"The device's LUN is served by node {dev.serving_node()}, not by " + f"{host.get_id()}") if tasks_controller.get_active_dev_restart_task(snode.cluster_id, dev.get_id()): raise ValueError("A restart task is active for the device") +def _rescan_size(snode, dev, path=None): + """Make the storage node's host re-read the size of the device's block + device (SCSI or NVMe rescan) and return it in bytes.""" + path = path or dev.by_id_path or dev.device_path + if not path: + raise ValueError("No block device path known for the device") + result, err = snode.client(timeout=60, retry=1).rescan_block_device(path) + if err or not result: + raise ValueError(f"Rescan of {path} failed: {err}") + return int(result["size"]) + + def _reopen_after_resize(rpc_client, dev): """Leave alceml's resize state with the backing bdev reopened. Returns True on success. On persistent failure alceml is told to fail its I/O, @@ -2198,44 +2348,15 @@ def _resize_failed(device_id, dev, msg, set_unavailable): return ValueError(msg) -def resize_device(device_id): - """Grow an lblk device online to the size its block device now has. - - The kernel is made to re-read the size first (SCSI or NVMe rescan on the - storage node's host). A smaller size is refused. Returns a dict with - ``device_id``, ``old_size``, ``new_size``, ``placement_size``, - ``resized``, and the cluster's ``uniform`` growth flag with a - ``warning`` when it is not uniform. Raises ValueError on refusal or - failure; the message says whether the device is unchanged.""" - db_controller = DBController() - try: - dev = db_controller.get_storage_device_by_id(device_id) - snode = db_controller.get_storage_node_by_id(dev.node_id) - except KeyError as e: - raise ValueError(f"Device not found: {device_id}") from e - _check_resizable(db_controller, dev, snode) - - path = dev.by_id_path or dev.device_path - if not path: - raise ValueError("No block device path known for the device") - result, err = snode.client(timeout=60, retry=1).rescan_block_device(path) - if err or not result: - raise ValueError(f"Rescan of {path} failed: {err}") - new_size = int(result["size"]) - old_size = dev.size - - if new_size < old_size: - raise ValueError( - f"The block device shrank from {old_size} to {new_size} bytes. Shrinking is not " - f"supported and destroys data: restore the LUN to at least {old_size} bytes") - if new_size == old_size: - uniform, _, _ = cluster_growth_uniformity(snode.cluster_id) - return {"device_id": device_id, "old_size": old_size, "new_size": new_size, - "placement_size": dev.weight_size(), "resized": False, "uniform": uniform, - "warning": ""} - - logger.info(f"Growing device {device_id} from {old_size} to {new_size} bytes") +def _reopen_alceml(snode, dev, base_bdev=None): + """Let alceml reopen its AIO bdev at the size the block device has now: + resize_begin -> bdev_aio_rescan (bdev_uring_rescan for a uring device) + -> resize_end, on ``snode`` (the node serving the device) with its base + bdev ``base_bdev`` (default: the owner's). Raises ValueError.""" + device_id = dev.get_id() + base_bdev = base_bdev or dev.nvme_bdev rpc_client = snode.rpc_client() + io_engine = utils.lblk_io_engine_of_device(snode, dev) try: rpc_client.alceml_bdev_backend_resize_begin( dev.alceml_bdev, request_timeout=RESIZE_RPC_TIMEOUT_SEC) @@ -2246,160 +2367,164 @@ def resize_device(device_id): rescan_error = None try: - rpc_client.bdev_aio_rescan(dev.nvme_bdev) + utils.rescan_lblk_bdev(rpc_client, base_bdev, io_engine) except RPCException as e: rescan_error = e if not _reopen_after_resize(rpc_client, dev): raise _resize_failed(device_id, dev, "alceml resize_end failed", True) if rescan_error is not None: - raise _resize_failed(device_id, dev, f"AIO rescan failed: {rescan_error}", False) + raise _resize_failed(device_id, dev, f"{io_engine} rescan failed: {rescan_error}", False) - def _apply(d, n=new_size): - if d.placement_size <= 0: - d.placement_size = d.size - d.size = n - _atomic_device_set(db_controller, snode.get_id(), device_id, _apply) - fresh = db_controller.get_storage_device_by_id(device_id) - device_events.device_resized(fresh, old_size, new_size) - - uniform, lo, hi = cluster_growth_uniformity(snode.cluster_id) - warning = "" - if not uniform: - warning = ( - f"The data devices of the cluster have grown unevenly (growth factors {lo:.2f} to {hi:.2f}). " - "Placement keeps filling every device in proportion to its original size, so the added " - "capacity is only fully usable once all data devices have grown by the same factor, or " - "after cluster grow-devices has updated the weights and rebalanced the cluster.") - logger.warning(warning) - return {"device_id": device_id, "old_size": old_size, "new_size": new_size, - "placement_size": fresh.weight_size(), "resized": True, "uniform": uniform, - "warning": warning} - - -#: Tasks that move data across the cluster map or change it. New weights are -#: never pushed while one of them is open. -_PLACEMENT_TASKS = ( - JobSchedule.FN_DEV_MIG, JobSchedule.FN_NEW_DEV_MIG, JobSchedule.FN_FAILED_DEV_MIG, - JobSchedule.FN_BALANCING_AFTER_NODE_RESTART, JobSchedule.FN_BALANCING_AFTER_DEV_REMOVE, - JobSchedule.FN_BALANCING_AFTER_DEV_EXPANSION, JobSchedule.FN_CLUSTER_EXPAND, - JobSchedule.FN_LVOL_MIG, JobSchedule.FN_LVOL_BATCH_MIG, -) +def _growth_result(grown, factor, devices, errors, reason, shrunk=()): + return {"grown": grown, "factor": round(factor, 6), "devices": devices, + "errors": errors, "reason": reason, "shrunk": list(shrunk)} -def _reweight_blocker(db_controller, cluster_id): - """Why new weights cannot be pushed now, or "" when they can. The new map - must reach every node before any data moves: a migration that runs while - nodes disagree about placement can lose writes.""" - cluster = db_controller.get_cluster_by_id(cluster_id) - if cluster.status != Cluster.STATUS_ACTIVE: - return f"the cluster is {cluster.status}, not active" - for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id): - if node.status == StorageNode.STATUS_REMOVED: - continue - if node.status != StorageNode.STATUS_ONLINE: - return f"storage node {node.get_id()} is {node.status}" - for dev in node.nvme_devices: - if dev.status in (NVMeDevice.STATUS_JM, NVMeDevice.STATUS_NEW, - NVMeDevice.STATUS_FAILED, NVMeDevice.STATUS_FAILED_AND_MIGRATED): - continue - if dev.status != NVMeDevice.STATUS_ONLINE: - return f"device {dev.get_id()} is {dev.status}" - for task in db_controller.get_job_tasks(cluster_id): - if (task.function_name in _PLACEMENT_TASKS and task.status != JobSchedule.STATUS_DONE - and not task.canceled): - return f"task {task.uuid} ({task.function_name}) is still open" - return "" - - -def rebalance_after_growth(cluster_id): - """Set the cluster-map weight of every online data device to its current - size, push the new map to every node, and start the cluster rebalance. - - Refused (nothing changed) unless the cluster is active, every node and - data device is online, and no migration or expansion is open. If a map - push fails, the new weights stay recorded with ``reweight_pending`` and - the rebalance is not started; the next call pushes again. Returns - ``{"weights_updated": bool, "started": bool, "reason": str}``: - ``weights_updated`` once every node runs on the new weights, ``started`` - once the rebalance is queued, ``reason`` when either is missing.""" + +def _device_report(dev, old_size): + return {"device_id": dev.get_id(), "old_size": old_size, "new_size": dev.size, + "backend_size": dev.backend_size, "unused_size": dev.unused_size(), + "placement_size": dev.weight_size()} + + +def grow_cluster_devices(cluster_id, lun_factor=None, lun_size=None): + """Grow every data device of an lblk cluster by the same factor (see + _grow_cluster_devices). + + ``lun_factor`` (multi-attach cluster with a registered array only): grow + every LUN the array plugin created by this factor at the array first and + wait until every node it is presented to sees the new size + (array_controller.grow_luns). If any LUN fails, nothing is grown on the + cluster; the uniform growth below would refuse anyway, since growth takes + the smallest factor of all block devices. The result then also carries + ``luns``. + + ``lun_size`` instead of ``lun_factor``: grow every LUN to that many bytes + (a LUN already that large is left alone, so a failed growth is simply + repeated) and make it the LUN size of nodes added later.""" + if lun_factor is None and lun_size is None: + return _grow_cluster_devices(cluster_id) + from simplyblock_core.controllers import array_controller + # Check before the array is touched: a refusal after the LUNs grew would + # leave them grown and the cluster not, and a re-run with the factor + # would grow them again. db_controller = DBController() cluster = db_controller.get_cluster_by_id(cluster_id) - blocker = _reweight_blocker(db_controller, cluster_id) - if blocker: - if cluster.reweight_pending: - return _rebalance_outcome(False, False, - f"New weights recorded, cluster map not pushed yet: {blocker}") - return _rebalance_outcome(False, False, f"Weights not updated: {blocker}") - - if not cluster.reweight_pending: - db_controller.atomic_update(cluster, lambda c: setattr(c, "reweight_pending", True)) - - def _apply(n): - for d in n.nvme_devices: - if d.status == NVMeDevice.STATUS_ONLINE: - d.placement_size = d.size - for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id): - if node.status != StorageNode.STATUS_REMOVED: - db_controller.atomic_update(node, _apply) - logger.info(f"Cluster {cluster_id}: device weights set to the current device sizes") - - nodes = [n for n in db_controller.get_storage_nodes_by_cluster_id(cluster_id) - if n.status == StorageNode.STATUS_ONLINE] - failed = [n.get_id() for n in nodes if not distr_controller.send_cluster_map_to_node(n)] - if failed: - return _rebalance_outcome( - False, False, f"New weights recorded, but the cluster map push failed on {', '.join(failed)}; " - "run cluster grow-devices again to retry") - - db_controller.atomic_update(db_controller.get_cluster_by_id(cluster_id), - lambda c: setattr(c, "reweight_pending", False)) - # The same rebalance as POST /clusters/{id}/rebalance: one expansion - # migration per distrib, under a balancing master task. - queued = bool(nodes) and tasks_controller.add_device_mig_task_for_node(nodes[0].get_id()) - device_events.devices_reweighted(cluster_id, queued) - if not queued: - return _rebalance_outcome( - True, False, "Weights updated, but no rebalance was queued (no volume store holds data, or a " - "node removal is recorded); start it with the cluster rebalance API when needed") - return _rebalance_outcome(True, True, "") - - -def _rebalance_outcome(weights_updated, started, reason): - return {"weights_updated": weights_updated, "started": started, "reason": reason} - - -def grow_cluster_devices(cluster_id, rebalance=True): - """Grow every online lblk data device of a cluster to the size of its - block device, one device at a time. Devices whose size did not change - are left alone; a device that fails does not stop the others. - - When the devices have grown unevenly and ``rebalance`` is set, the - weights are updated and the cluster is rebalanced (rebalance_after_growth), - unless a device failed to grow. Returns ``{"devices": [...], "errors": - {...}, "uniform": bool, "rebalance": {...} | None}`` (see - rebalance_after_growth for the rebalance dict); - ``uniform`` describes the growth before any weight change.""" + if cluster.device_mode == constants.DEVICE_MODE_LBLK: + errors = _resizable_errors(db_controller, cluster, _growth_devices(db_controller, cluster_id)) + if errors: + result = _growth_result(False, 1.0, [], errors, + "Not every data device can be grown; no LUN was grown at " + "the array") + result["luns"] = {} + return result + lun_report = array_controller.grow_luns(cluster_id, lun_factor, size=lun_size) + if lun_report["errors"]: + result = _growth_result(False, 1.0, [], dict(lun_report["errors"]), + "Not every LUN could be grown at the array; nothing was grown " + "on the cluster") + else: + result = _grow_cluster_devices(cluster_id) + result["luns"] = lun_report["luns"] + return result + + +def _grow_cluster_devices(cluster_id): + """Grow every data device of an lblk cluster by the same factor: the + smallest growth of any of their block devices (see the section comment). + + Every data device must be online and growable; otherwise nothing grows. + All block devices are rescanned first and their sizes recorded. If one + of them is smaller than its device, the shrink is reported (event and + device_shrunk alert) and nothing grows. Otherwise every alceml reopens + its grown block device, and only when all of them succeeded is every + device size set to ``size * factor``; a failure in between leaves all + recorded sizes unchanged, so the cluster stays uniform and a re-run + finishes the job. Repeated runs without new growth change nothing. + + Returns ``{"grown": bool, "factor": float, "devices": [...], + "errors": {device_id: message}, "reason": str, "shrunk": [device_id]}``; + every device entry has + ``device_id``, ``old_size``, ``new_size``, ``backend_size``, + ``unused_size`` and ``placement_size``.""" db_controller = DBController() - results, errors = [], {} - for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id): - if node.is_secondary_node: - continue - for dev in node.nvme_devices: - if dev.status != NVMeDevice.STATUS_ONLINE or dev.bdev_type != "aio": - continue - try: - results.append(resize_device(dev.get_id())) - except ValueError as e: - errors[dev.get_id()] = str(e) - uniform, _, _ = cluster_growth_uniformity(cluster_id) - outcome = None - pending = db_controller.get_cluster_by_id(cluster_id).reweight_pending - if rebalance and (not uniform or pending): - if errors and not pending: - outcome = _rebalance_outcome( - False, False, "Weights not updated: some devices failed to grow; fix them and run " - "cluster grow-devices again") - else: - outcome = rebalance_after_growth(cluster_id) - return {"devices": results, "errors": errors, "uniform": uniform, "rebalance": outcome} + cluster = db_controller.get_cluster_by_id(cluster_id) + if cluster.device_mode != constants.DEVICE_MODE_LBLK: + raise ValueError("Online resizing is only supported for Linux block devices (lblk device mode)") + pairs = _growth_devices(db_controller, cluster_id) + if not pairs: + return _growth_result(False, 1.0, [], {}, "The cluster has no data devices") + + errors = _resizable_errors(db_controller, cluster, pairs) + if errors: + return _growth_result(False, 1.0, [], errors, + "Not every data device can be grown; growth is uniform across all " + "data devices, so nothing was grown") + + hosts = {dev.get_id(): _growth_host(db_controller, node, dev) for node, dev in pairs} + backend = {} + for node, dev in pairs: + host, _, path = hosts[dev.get_id()] + try: + backend[dev.get_id()] = _rescan_size(host, dev, path) + except ValueError as e: + errors[dev.get_id()] = str(e) + if errors: + return _growth_result(False, 1.0, [], errors, + "Not every block device could be rescanned; nothing was grown") + + for node, dev in pairs: + size = backend[dev.get_id()] + if dev.backend_size != size: + _atomic_device_set(db_controller, node.get_id(), dev.get_id(), + lambda d, s=size: setattr(d, "backend_size", s)) + dev.backend_size = size + shrunk = [(node, dev) for node, dev in pairs if dev.is_shrunk()] + if shrunk: + for _, dev in shrunk: + device_events.device_shrink_detected(dev, dev.backend_size) + errors[dev.get_id()] = ( + f"The block device shrank to {dev.backend_size} bytes, below the device size of " + f"{dev.size} bytes. Shrinking is not supported: restore the LUN to at least the " + f"device size") + return _growth_result(False, 1.0, [], errors, + "A block device is smaller than its device; nothing was grown", + shrunk=[dev.get_id() for _, dev in shrunk]) + + factor = min(backend[dev.get_id()] / dev.size for _, dev in pairs) + if factor < GROWTH_MIN_FACTOR: + return _growth_result(False, 1.0, [_device_report(dev, dev.size) for _, dev in pairs], {}, + "Not every block device has grown; growth is uniform across all data " + "devices, so nothing was grown") + + targets = {} + for _, dev in pairs: + target = int(dev.size * factor) // GROWTH_SIZE_ALIGN * GROWTH_SIZE_ALIGN + targets[dev.get_id()] = min(max(target, dev.size), backend[dev.get_id()]) + + logger.info(f"Growing {len(pairs)} data devices of cluster {cluster_id} by factor {factor:.4f}") + for node, dev in pairs: + host, base, _ = hosts[dev.get_id()] + try: + _reopen_alceml(host, dev, base) + except ValueError as e: + errors[dev.get_id()] = str(e) + return _growth_result(False, factor, [], errors, + "A device failed to grow; all device sizes are unchanged, run " + "grow-devices again once the device is back") + + reports = [] + for node, dev in pairs: + old_size = dev.size + new_size = targets[dev.get_id()] + + def _apply(d, n=new_size): + if d.placement_size <= 0: + d.placement_size = d.size + d.size = n + _atomic_device_set(db_controller, node.get_id(), dev.get_id(), _apply) + fresh = db_controller.get_storage_device_by_id(dev.get_id()) + device_events.device_resized(fresh, old_size, new_size) + reports.append(_device_report(fresh, old_size)) + return _growth_result(True, factor, reports, {}, "") diff --git a/simplyblock_core/controllers/device_events.py b/simplyblock_core/controllers/device_events.py index c2278ce666..2e278feda2 100644 --- a/simplyblock_core/controllers/device_events.py +++ b/simplyblock_core/controllers/device_events.py @@ -106,18 +106,24 @@ def device_resized(device, old_size, new_size, caused_by=ec.CAUSED_BY_CLI): caused_by, ec.EVENT_STATUS_CHANGE) -def devices_reweighted(cluster_id, rebalancing, caused_by=ec.CAUSED_BY_CLI): - """Cluster-log entry for new device weights after an uneven growth of - lblk devices; the rebalance that follows is logged by its tasks.""" +def device_shrink_detected(device, backend_size, caused_by=ec.CAUSED_BY_CLI): + """ERROR-level cluster-log entry for a block device found smaller than + its storage device. The device_shrunk alert stands until a rescan sees + the block device at its full size again.""" + db_controller = DBController() + snode = db_controller.get_storage_node_by_id(device.node_id) ec.log_event_cluster( - cluster_id=cluster_id, + cluster_id=snode.cluster_id, domain=ec.DOMAIN_CLUSTER, event=ec.EVENT_STATUS_CHANGE, - db_object=DBController().get_cluster_by_id(cluster_id), + db_object=device, caused_by=caused_by, - message="Device weights set to the current device sizes after uneven growth" - + ("; rebalancing started" if rebalancing else ""), - node_id=cluster_id) + message=(f"Block device of the device shrank to {utils.humanbytes(backend_size)}, " + f"below the device size of {utils.humanbytes(device.size)}. Shrinking is not " + f"supported: restore the LUN to at least the device size"), + node_id=device.get_id(), + event_level=EventObj.LEVEL_ERROR, + storage_id=device.cluster_device_order) def device_resize_failed(device, message, caused_by=ec.CAUSED_BY_CLI): diff --git a/simplyblock_core/controllers/events_controller.py b/simplyblock_core/controllers/events_controller.py index b0d5c7dec2..7450e9fded 100644 --- a/simplyblock_core/controllers/events_controller.py +++ b/simplyblock_core/controllers/events_controller.py @@ -13,12 +13,20 @@ EVENT_OBJ_CREATED = "OBJ_CREATED" EVENT_OBJ_DELETED = "OBJ_DELETED" EVENT_CAPACITY = "CAPACITY" +# The cluster's online resources are below an erasure-coding floor +# (simplyblock_core.floors); the message names the floor. +EVENT_FLOOR = "FLOOR" EVENT_LIMIT_REACHED = "LIMIT_REACHED" #: A distrib reported an IO it could not serve from the chunks it can reach. EVENT_DISTR_IO_ERROR = "DISTR_IO_ERROR" #: A client LVS port was reopened FENCE_RELEASE_BUDGET times in one window #: and SPDK fenced it again each time; the control plane stopped reopening it. EVENT_FENCE_LOOP = "FENCE_LOOP" +#: Multi-attach array LUN mode: a LUN was (or could not be) taken over by +#: another node after its serving node failed. +EVENT_LUN_TAKEOVER = "LUN_TAKEOVER" +# Multi-attach array LUN lifecycle (controllers/array_controller). +EVENT_ARRAY_LUN = "ARRAY_LUN" #: A replication step on the target keeps failing (e.g. the landing merge #: cannot get memory); the control plane backs off and keeps retrying. EVENT_REPLICATION_STALLED = "REPLICATION_STALLED" diff --git a/simplyblock_core/controllers/lun_takeover.py b/simplyblock_core/controllers/lun_takeover.py new file mode 100644 index 0000000000..73df44fb4a --- /dev/null +++ b/simplyblock_core/controllers/lun_takeover.py @@ -0,0 +1,1349 @@ +"""Multi-attach array LUN mode, phase 3: take a LUN over on a standby node. + +When the node serving a multi-attach device (``NVMeDevice.serving_node()``) +fails, its peers' controllers ``remote_alceml_`` hold I/O for +``MULTI_ATTACH_HOLD_SEC``. Within that window :func:`take_over_lun`: + +1. fences the failed node from the LUN at the array -- before anything else: + its host multipath layer may still hold queued writes that would land late + and overwrite newer data; +2. picks a standby: the owner's LVS secondary, else its tertiary, else the + owner itself; ONLINE and not fenced from this LUN; +3. verifies through the standby's node agent that it sees the LUN and that + block 0 carries the alceml signature (read-only); 2 and 3 run while the + fence is applied, nothing is built on the standby before its ack; +4. on the standby builds the base bdev, the alceml OPEN-ONLY + (``pba_init_mode=1``; 2/3 would format the LUN and destroy its data), the + PT bdev and the device subsystem under the same NQN, with the standby's + controller-ID range, the same namespace UUID and the standby's listeners; +5. records the standby as the serving node (atomically; the device stays + ONLINE, no unavailable broadcast, no cluster-map change); +6. adds the standby's export as a path of ``remote_alceml_`` on every + ONLINE node (the standby too: its loopback), all in parallel, where the + held I/O resumes; +7. detaches the failed node's path from those controllers so it never + reconnects. + +The record is written before the paths are added (design order: paths, then +record): the export on the standby is what serves the LUN from then on, and +any node that misses its path in step 6 repairs it from the record +(connect_device uses the serving node). A re-run is idempotent: each step +checks what is already in place, and a LUN already served by an ONLINE node +other than the failed one only gets steps 6-7. + +Never done: a takeover without a configured array plugin or without the +array's acknowledgement of the fence; an alceml create with any init mode but +open-only. When no standby can read the LUN, nothing is changed: peers keep +holding until their hold expires, and the caller marks the device +unavailable. + +Warm standby (opt-in, ``Cluster.multi_attach_warm_standby``): every attached +non-serving node keeps a standby alceml of the LUN open (``standby=true``, +never exported) whose page map the data plane keeps warm +(:func:`ensure_warm_standbys`, driven by the storage-node monitor). Step 4 +then reuses that instance instead of opening the LUN cold (a full page-map +load, minutes at 100 TiB): it only adds the PT bdev, subsystem, listeners and +namespace. A standby activates on its first I/O (fence by preemption, epoch +bump, final refresh), so the takeover waits for the activation after step 6, +when the peers' held I/O reaches it. If it does not activate, the failed node +is fenced at the array, the export on the standby is torn down and redone +cold (the same NQN and IPs: the paths already point there). +""" + +from __future__ import annotations + +import logging +import os +import socket +import threading +import time +import uuid as uuid_mod +from concurrent.futures import ThreadPoolExecutor + +from simplyblock_core import array, constants, multi_attach, utils +from simplyblock_core.controllers import events_controller as ec +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.events import EventObj +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode + +logger = logging.getLogger() + +#: bdev_alceml_create pba_init_mode that opens an existing alceml without +#: touching its data. The only mode a takeover ever uses. +ALCEML_OPEN_ONLY = 1 + +#: Device states a takeover acts on. A failed, removed or new device is not +#: taken over. +_TAKEOVER_STATES = (NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE, + NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE) + +_local_locks: dict[str, threading.Lock] = {} +# (device_id, node_id) of array fences still running after a fence by +# reservation: the node is not unfenced (reintegrated) meanwhile. +_fences_in_flight: set = set() +_fences_guard = threading.Lock() +_local_locks_guard = threading.Lock() + + +#: Outcomes of :func:`take_over_lun_outcome`. +TAKEN_OVER = "taken_over" # served by a live node other than the failed one +IN_PROGRESS = "in_progress" # another takeover of this LUN is running +NO_STANDBY = "no_standby" # fenced, but no standby can read the LUN: hold +FAILED = "failed" # not taken over (no plugin, fence, export, ...) + + +class TakeoverError(Exception): + """A takeover step failed; the message says which and why.""" + + +class StandbyUnreachable(TakeoverError): + """The standby's SPDK stopped answering during the export: the next + standby is tried at once (no retry on it, no array fence first).""" + + +#: Consecutive connection errors while waiting for an activation after which +#: the standby counts as gone (the poll interval is 50 ms, each call times out +#: after 1 s). +ACTIVATION_CONNECTION_ERRORS = 2 +#: Bound of one standby candidate's LUN inventory query (no retries). +STANDBY_PROBE_TIMEOUT_SEC = 3.0 +#: How long a candidate that reads the LUN waits for an earlier candidate +#: (candidate order) whose probe has not answered yet. +STANDBY_ORDER_GRACE_SEC = 0.5 + + +def _local_lock(device_id) -> threading.Lock: + with _local_locks_guard: + return _local_locks.setdefault(device_id, threading.Lock()) + + +def _event(cluster_id, device, message, level=EventObj.LEVEL_INFO): + try: + ec.log_event_cluster( + cluster_id=cluster_id, + domain=ec.DOMAIN_CLUSTER, + event=ec.EVENT_LUN_TAKEOVER, + db_object=device, + caused_by=ec.CAUSED_BY_MONITOR, + message=message, + node_id=device.get_id(), + event_level=level, + storage_id=device.cluster_device_order) + except Exception as e: # noqa: BLE001 + logger.error("LUN takeover event of %s not logged: %s", device.get_id(), e) + + +def _check_deadline(deadline, step): + if time.monotonic() > deadline: + raise TakeoverError( + f"deadline of {constants.MULTI_ATTACH_TAKEOVER_DEADLINE_SEC}s passed before {step}") + + +def _remaining(deadline, cap): + return max(1, min(cap, int(deadline - time.monotonic()))) + + +def _transport(node): + if node.active_rdma: + return "RDMA" + if node.active_tcp: + return "TCP" + return None + + +def _device_on_owner(owner, device_id): + for dev in owner.nvme_devices: + if dev.get_id() == device_id: + return dev + return None + + +def take_over_lun(device_id, failed_node_id, reason="", *, deadline_sec=None) -> bool: + """Move the LUN of multi-attach device ``device_id`` off the failed node + ``failed_node_id`` onto a standby (see the module docstring). + + Returns True when the device is served by an ONLINE node other than the + failed one afterwards (taken over now, or already before), False when it + was not taken over (the reason is logged and raised as a cluster event) + or another takeover of it is running. Never raises. + """ + return take_over_lun_outcome(device_id, failed_node_id, reason, + deadline_sec=deadline_sec) == TAKEN_OVER + + +def take_over_lun_outcome(device_id, failed_node_id, reason="", *, deadline_sec=None) -> str: + """:func:`take_over_lun`, answering one of TAKEN_OVER, IN_PROGRESS, + NO_STANDBY or FAILED.""" + db = DBController() + deadline = time.monotonic() + (deadline_sec if deadline_sec is not None + else constants.MULTI_ATTACH_TAKEOVER_DEADLINE_SEC) + try: + dev = db.get_storage_device_by_id(device_id) + owner = db.get_storage_node_by_id(dev.node_id) + cluster = db.get_cluster_by_id(owner.cluster_id) + except KeyError as e: + logger.error("LUN takeover of %s: %s", device_id, e) + return FAILED + if not (multi_attach.is_multi_attach(cluster) and multi_attach.is_multi_attach(dev)): + logger.error("LUN takeover of %s refused: not a multi-attach device", device_id) + return FAILED + + local = _local_lock(device_id) + if not local.acquire(blocking=False): + logger.info("LUN takeover of %s already running in this process", device_id) + return IN_PROGRESS + token = f"{socket.gethostname()}:{os.getpid()}:{uuid_mod.uuid4().hex[:8]}" + try: + try: + held, holder = db.acquire_lun_takeover_lock( + device_id, token, constants.MULTI_ATTACH_TAKEOVER_LOCK_TTL_SEC) + except Exception as e: # noqa: BLE001 + logger.error("LUN takeover of %s: lease not taken: %s", device_id, e) + return FAILED + if not held: + logger.info("LUN takeover of %s already running (%s)", device_id, holder) + return IN_PROGRESS + try: + return _take_over_locked(db, cluster, device_id, failed_node_id, reason, deadline) + finally: + try: + db.release_lun_takeover_lock(device_id, token) + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover of %s: lease not released: %s", device_id, e) + finally: + local.release() + + +def _take_over_locked(db, cluster, device_id, failed_node_id, reason, deadline) -> str: + owner = db.get_storage_node_by_id(db.get_storage_device_by_id(device_id).node_id) + dev = _device_on_owner(owner, device_id) + if dev is None: + logger.error("LUN takeover of %s: device not on its owner %s", device_id, owner.get_id()) + return FAILED + cluster_id = owner.cluster_id + logger.info("LUN takeover of %s (LUN %s, owner %s) from node %s: %s", + device_id, dev.lun_wwn, owner.get_id(), failed_node_id, reason or "node lost") + + if dev.status not in _TAKEOVER_STATES: + logger.info("LUN takeover of %s skipped: device is %s", device_id, dev.status) + return FAILED + + nodes_by_id = {n.get_id(): n for n in db.get_storage_nodes_by_cluster_id(cluster_id)} + serving_id = dev.serving_node() + if serving_id != failed_node_id: + # Re-run, or a stale trigger: only complete the paths if the LUN is + # served by a live node. + server = nodes_by_id.get(serving_id) + if server is None or server.status != StorageNode.STATUS_ONLINE: + logger.warning("LUN takeover of %s: served by %s (%s), not by %s; nothing done", + device_id, serving_id, getattr(server, "status", "unknown"), failed_node_id) + return FAILED + logger.info("LUN takeover of %s: already served by %s; completing its paths", + device_id, serving_id) + failed = nodes_by_id.get(failed_node_id) + _switch_paths(db, cluster_id, dev, server, failed, _failed_paths(failed, None, None)) + return TAKEN_OVER + + failed = nodes_by_id.get(failed_node_id) + if failed is None: + logger.error("LUN takeover of %s: failed node %s not found", device_id, failed_node_id) + return FAILED + old_ips = [ip.strip() for ip in (dev.nvmf_ip or "").split(",") if ip.strip()] + old_port = dev.nvmf_port + + # 2 + 3 run while the fence is applied: choosing the standby only reads + # (the node agent's LUN inventory), and nothing is built on it before + # the array has acknowledged the fence. Every volume of a 1+0 cluster + # waits on this LUN. + pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix=f"standby-{device_id[:8]}") + choice = pool.submit(_choose_standby, owner, dev, failed_node_id, nodes_by_id, deadline) + pool.shutdown(wait=False) + by_reservation = False + try: + # 1. Fence -- before anything else is changed. With reservations the + # standby's alceml fences at open (preempting the failed node's + # reservation, step 4) and the array fence follows off the I/O path. + by_reservation = _fence_by_reservation(cluster, dev) + if not by_reservation: + _fence(cluster, dev, failed) + _record_fenced(db, owner, device_id, failed_node_id) + _check_deadline(deadline, "choosing a standby") + + # 2 + 3. A standby that can read the LUN. + standby, path, why = choice.result(timeout=max(1.0, deadline - time.monotonic())) + if standby is None: + msg = (f"LUN {dev.lun_wwn} of device {device_id} not taken over from node " + f"{failed_node_id}: no standby can read it ({why}); holding I/O until " + f"the peers' hold expires") + logger.error(msg) + _event(cluster_id, dev, msg, EventObj.LEVEL_CRITICAL) + return NO_STANDBY + # 4. Open-only export on the standby. A standby that fails and no + # longer answers is left for the next candidate (2026-11-20 FTT2 bed: + # the standby chosen was rebooting; the takeover failed although a + # third node could read the LUN, and the 1+0 cluster suspended). + tried, retried = set(), set() + while True: + _check_deadline(deadline, f"exporting the LUN on {standby.get_id()}") + try: + export = _export_on_standby(db, cluster, standby, dev, path, deadline) + break + except TakeoverError as e: + sid = standby.get_id() + gone = isinstance(e, StandbyUnreachable) + if by_reservation and not gone: + # The standby answers but did not take the reservation: + # fence at the array. A standby that is gone fenced + # nothing; the next one opens with its own key and + # preempts, so it is tried at once. + logger.warning("LUN takeover of %s: fence by reservation failed (%s); " + "fencing at the array", device_id, e) + by_reservation = False + _fence(cluster, dev, failed) + _record_fenced(db, owner, device_id, failed_node_id) + if not gone and sid not in retried and _reachable(standby): + retried.add(sid) + continue + tried.add(sid) + logger.warning("LUN takeover of %s: export on standby %s failed (%s); trying " + "another standby", device_id, sid, e) + _check_deadline(deadline, "choosing another standby") + standby, path, why = _choose_standby(owner, dev, failed_node_id, nodes_by_id, + deadline, exclude=tried) + if standby is None: + raise TakeoverError(f"export failed on {sorted(tried)} and no other standby " + f"can read the LUN ({why}): {e}") + if by_reservation: + # Fenced by the reservation: recorded now, detached at the array + # in the background (the attachment reconcile detaches it too). + _record_fenced(db, owner, device_id, failed_node_id) + _fence_in_background(cluster, dev, failed) + _check_deadline(deadline, "recording the takeover") + + # 5 (design step 6). Record, atomically. + if not _record_serving(db, owner, device_id, failed_node_id, standby, path, export): + raise TakeoverError("the device record changed under the takeover") + except TakeoverError as e: + msg = f"LUN takeover of device {device_id} from node {failed_node_id} failed: {e}" + logger.error(msg) + _event(cluster_id, dev, msg, EventObj.LEVEL_CRITICAL) + return FAILED + except Exception as e: # noqa: BLE001 + msg = f"LUN takeover of device {device_id} from node {failed_node_id} failed: {e}" + logger.exception(msg) + _event(cluster_id, dev, msg, EventObj.LEVEL_CRITICAL) + return FAILED + + dev = _device_on_owner(db.get_storage_node_by_id(owner.get_id()), device_id) or dev + # 6 + 7 (design steps 5 and 7). New path on every ONLINE node, old one off. + added = _switch_paths(db, cluster_id, dev, standby, failed, + _failed_paths(failed, old_ips, old_port)) + if export.get("warm") and not _activate_warm_standby( + db, cluster, owner, dev, standby, failed, path, export, deadline): + return FAILED + + if dev.status != NVMeDevice.STATUS_ONLINE: + # The hold had expired and the device was marked unavailable; it is + # served again. + from simplyblock_core.controllers import device_controller + device_controller.device_set_state( + device_id, NVMeDevice.STATUS_ONLINE, + cause=device_controller.CAUSE_LUN_TAKEOVER, connect_peers=False) + + msg = (f"LUN {dev.lun_wwn} of device {device_id} taken over from node {failed_node_id} " + f"by node {standby.get_id()} ({path}); new path on {len(added)} node(s)") + logger.info(msg) + _event(cluster_id, dev, msg, EventObj.LEVEL_WARN) + return TAKEN_OVER + + +# ---------------------------------------------------------------- 1. fence + +def _fence(cluster, dev, failed): + # The array's own ids when the plugin created the LUN / registered the + # host (array_controller), else the WWN / hostname as in phase 3. + host = multi_attach.array_host_ref(failed) + if not host: + raise TakeoverError(f"node {failed.get_id()} has no hostname to fence it by") + lun = multi_attach.array_lun_ref(dev) + if not lun: + raise TakeoverError("the device has no LUN WWN to fence") + ok = None + for fresh in (False, True): + if fresh: + # The cached connection may have gone stale (expired session). + array.forget_plugin(cluster.get_id()) + try: + plugin = array.cached_plugin_for_cluster(cluster) + except array.ArrayNotConfigured: + raise TakeoverError( + "no array plugin is configured (cluster array_plugin); a LUN is never " + "taken over without fencing its serving node") + except array.ArrayError as e: + raise TakeoverError(f"array plugin unavailable: {e}") + try: + ok = plugin.fence(lun, host) + break + except Exception as e: # noqa: BLE001 + if fresh: + raise TakeoverError(f"array fence of {host} on LUN {lun} failed: {e}") + logger.warning("LUN %s: fence of %s failed (%s); again on a new array connection", + lun, host, e) + if not ok: + raise TakeoverError(f"array did not acknowledge the fence of {host} on LUN {lun}") + logger.info("LUN %s: node %s (%s) fenced at the array", lun, failed.get_id(), host) + _forget_attachment(dev.get_id(), failed.get_id()) + + +def _forget_attachment(device_id, node_id): + """The fence detached ``node_id`` from the LUN: drop it from the + device's ``array_attached_node_ids`` so the attachment reconcile, which + calls the array only for nodes it does not list, presents the LUN to it + again once the record no longer fences it. Best-effort.""" + try: + db = DBController() + dev = db.get_storage_device_by_id(device_id) + owner = db.get_storage_node_by_id(dev.node_id) + except Exception as e: # noqa: BLE001 + logger.warning("LUN fence of %s on %s: attachment record not updated: %s", + node_id, device_id, e) + return + + def _mut(n): + d = _device_on_owner(n, device_id) + if d is None or node_id not in (d.array_attached_node_ids or []): + return False + d.array_attached_node_ids = [x for x in d.array_attached_node_ids if x != node_id] + return True + try: + db.atomic_update(owner, _mut) + except Exception as e: # noqa: BLE001 + logger.warning("LUN fence of %s on %s: attachment record not updated: %s", + node_id, device_id, e) + + +def _fence_by_reservation(cluster, dev) -> bool: + """Whether this takeover fences by persistent reservation: the device's + alceml instances run with keys (NVMeDevice.lun_reservations, decided + with the array's capability when the LUN was added).""" + return multi_attach.reservations_enabled(cluster, dev) + + +def _fence_in_background(cluster, dev, failed): + """The array fence after a fence by reservation: off the I/O path, for + good measure (the reservation already rejects the node's writes).""" + key = (dev.get_id(), failed.get_id()) + + def _run(): + try: + _fence(cluster, dev, failed) + except TakeoverError as e: + logger.warning("LUN %s: array fence of %s after the reservation failed: %s; the " + "attachment reconcile detaches it", dev.lun_wwn, failed.get_id(), e) + finally: + with _fences_guard: + _fences_in_flight.discard(key) + with _fences_guard: + _fences_in_flight.add(key) + threading.Thread(target=_run, name=f"fence-{dev.get_id()[:8]}", daemon=True).start() + + +def fence_in_flight(device_id, node_id) -> bool: + """Whether an array fence of ``node_id`` on the LUN of ``device_id`` is + still running in this process.""" + with _fences_guard: + return (device_id, node_id) in _fences_in_flight + + +def _wait_active(rpc, alceml, deadline): + """Wait until ``alceml`` (created with a reservation key, not standby) + holds the reservation and serves. Raises TakeoverError.""" + limit = min(deadline, time.monotonic() + constants.MULTI_ATTACH_ACTIVATION_WAIT_SEC) + last = None + unreachable = 0 + while True: + try: + last = rpc.alceml_get_role(alceml) or {} + unreachable = 0 + except Exception as e: # noqa: BLE001 + last = {"error": str(e)} + unreachable += 1 + if unreachable >= ACTIVATION_CONNECTION_ERRORS: + # The standby is gone (2026-11-20 FTT2 bed: a rebooting + # standby cost the whole wait, 5 s, of a 13 s stall). + raise StandbyUnreachable(f"alceml {alceml}: SPDK not answering ({e})") + if last.get("role") == "active" and last.get("state") == "active": + logger.info("LUN takeover: %s holds the reservation (epoch %s, activation %s ms)", + alceml, last.get("epoch"), last.get("last_activation_ms")) + return + if last.get("state") == "act_failed" or time.monotonic() >= limit: + raise TakeoverError(f"alceml {alceml} did not take the reservation: {last}") + time.sleep(0.05) + + +def _record_fenced(db, owner, device_id, node_id): + def _mut(n): + d = _device_on_owner(n, device_id) + if d is None or node_id in d.fenced_node_ids: + return False + d.fenced_node_ids = list(d.fenced_node_ids) + [node_id] + return True + db.atomic_update(owner, _mut) + + +# --------------------------------------------------------- 2 + 3. standby + +def _choose_standby(owner, dev, failed_node_id, nodes_by_id, deadline, exclude=()): + reasons: list = [] + eligible: list = [] + for nid in multi_attach.standby_candidates(owner, failed_node_id): + node = nodes_by_id.get(nid) + if node is None or nid in exclude: + continue + if node.status != StorageNode.STATUS_ONLINE: + reasons.append(f"{nid}: {node.status}") + continue + if nid in (dev.fenced_node_ids or []): + reasons.append(f"{nid}: fenced from this LUN") + continue + if _transport(node) is None: + reasons.append(f"{nid}: no active fabric") + continue + eligible.append((nid, node)) + if not eligible: + return None, None, "; ".join(reasons) or "no standby candidate" + # Every candidate is probed at once, each call bounded and without + # retries; the first in candidate order that reads the LUN wins. One + # after the other, a candidate that had just lost its network held the + # choice until the takeover deadline (2026-11-20 FTT2 bed, network cuts + # of two nodes: the LUN went unavailable and the cluster suspended). + pool = ThreadPoolExecutor(max_workers=len(eligible), thread_name_prefix=f"probe-{dev.get_id()[:8]}") + probes = [(nid, node, pool.submit(_probe, node, dev, deadline)) for nid, node in eligible] + pool.shutdown(wait=False) + # The wait is bounded too (a probe that ignores its client timeout must + # not hold the choice). Candidate order wins, but a candidate that already + # reads the LUN is not kept waiting more than STANDBY_ORDER_GRACE_SEC for + # an earlier one that does not answer. + now = time.monotonic() + probes_done_by = min(deadline, now + STANDBY_PROBE_TIMEOUT_SEC + 0.5) + first_ready_at = None + while True: + now = time.monotonic() + best = None + pending_before = False + for nid, node, fut in probes: + if not fut.done(): + pending_before = True + continue + try: + path, _why = fut.result() + except Exception: # noqa: BLE001 + path = None + if path: + best = (nid, node, path) + break + if best and not pending_before: + break + if best and first_ready_at is None: + first_ready_at = now + if best and now - first_ready_at >= STANDBY_ORDER_GRACE_SEC: + break + if now >= probes_done_by or all(f.done() for _n, _o, f in probes): + break + time.sleep(0.02) + if best: + nid, node, path = best + logger.info("LUN %s: standby %s reads it at %s", dev.lun_wwn, nid, path) + return node, path, "" + for nid, _node, fut in probes: + if not fut.done(): + reasons.append(f"{nid}: probe did not answer within {STANDBY_PROBE_TIMEOUT_SEC:.0f}s") + continue + try: + _path, why = fut.result() + except Exception as e: # noqa: BLE001 + why = f"agent query failed: {e}" + reasons.append(f"{nid}: {why}") + return None, None, "; ".join(reasons) or "no standby candidate" + + +def _reachable(node) -> bool: + """Whether the node's SPDK answers right now (no retries).""" + try: + node.rpc_client(timeout=constants.MULTI_ATTACH_PROBE_TIMEOUT_SEC, retry=0).get_version() + return True + except Exception: # noqa: BLE001 + return False + + +def _probe(node, dev, deadline): + try: + result, err = node.client(timeout=min(_remaining(deadline, 15), STANDBY_PROBE_TIMEOUT_SEC), + retry=0, connect_retry=0).lun_info(dev.lun_wwn) + except Exception as e: # noqa: BLE001 + return None, f"agent query failed: {e}" + if err: + return None, f"agent: {err}" + if not isinstance(result, list): + return None, "agent returned no LUN inventory" + return multi_attach.pick_lun_path(result, dev) + + +# --------------------------------------------------------------- 4. export + +def _create_base_bdev(rpc, name, path, dev): + """The base bdev over the LUN on the standby, with the block size the + device was formatted with (NVMeDevice.aio_block_size; 0 = legacy, SPDK's + logical block size) and checked after creation: alceml over a base bdev + of another block size cannot read the device. The engine follows the + standby's own path to the LUN (a device-mapper node there needs io_uring + even if the owner reaches the LUN as a plain disk), else what the device + records.""" + engine = utils.lblk_io_engine_for({"path": path}) + if engine != constants.LBLK_IO_ENGINE_URING: + engine = dev.lblk_io_engine or constants.LBLK_IO_ENGINE_AIO + utils.create_aio_bdev(rpc, name, path, block_size=int(dev.aio_block_size or 0), + new_device=False, verify=True, io_engine=engine) + return True + + +def _alceml_open_kwargs(cluster, dev, extra) -> dict: + """``create_alceml`` kwargs of an open-only alceml of ``dev`` (takeover, + warm standby): never formats, the cluster's page and checksum settings, + plus ``extra`` (multi_attach.alceml_create_kwargs).""" + checksum_method, cache_size, cache_eviction = utils.alceml_checksum_params(cluster, dev) + return dict( + pba_init_mode=ALCEML_OPEN_ONLY, + write_protection=cluster.distr_ndcs > 1, + pba_page_size=cluster.page_size_in_blocks, + full_page_unmap=cluster.full_page_unmap, + checksum_method=checksum_method, + cache_size=cache_size, + cache_eviction_threshold=cache_eviction, + force_4k_atomic=getattr(cluster, 'atomic_4k', False), + **extra, + ) + + +def _alceml_role(node, alceml) -> dict: + """``alceml_get_role`` of ``alceml`` on ``node`` (fast, no retries); {} + when it does not answer.""" + try: + return node.rpc_client(timeout=1, retry=0).alceml_get_role(alceml) or {} + except Exception as e: # noqa: BLE001 + logger.info("alceml_get_role of %s on %s failed: %s", alceml, node.get_id(), e) + return {} + + +def _is_standby_instance(role) -> bool: + """A standby alceml instance that can still activate on its first I/O + (opened standby, or an active instance that was deposed).""" + return role.get("role") == "standby" and role.get("state") != "act_failed" + + +def _reuse_warm_standby(rpc, standby, alceml): + """The standby node holds ``alceml`` already: ``(warm, exists)``. A + standby instance is reused as is (warm). One whose activation failed is + closed, so that it is opened again active; anything else (an active + instance) is left to the regular path.""" + role = _alceml_role(standby, alceml) + if _is_standby_instance(role): + logger.info("LUN takeover: reusing the warm standby %s on %s (warm %s, %s refreshes)", + alceml, standby.get_id(), role.get("warm"), role.get("refreshes_count")) + return True, True + if role.get("role") == "standby": + logger.warning("LUN takeover: standby %s on %s failed its activation (%s); reopening it", + alceml, standby.get_id(), role) + _teardown(rpc, standby.get_id(), [("alceml", alceml)]) + if rpc.bdev_get(alceml): + raise TakeoverError(f"failed standby alceml {alceml} not closed on {standby.get_id()}") + return False, False + return False, True + + +def _export_on_standby(db, cluster, standby, dev, path, deadline, allow_warm=True) -> dict: + if getattr(dev, "bdev_type", "aio") != "aio": + raise TakeoverError(f"device type {dev.bdev_type!r} cannot be taken over (array LUNs are lblk)") + rpc = standby.rpc_client(timeout=_remaining(deadline, 60)) + sid = standby.get_id() + created = [] + try: + base = utils.aio_bdev_name_for_serial(dev.serial_number) + info = rpc.bdev_get(base) + if not info: + if not _create_base_bdev(rpc, base, path, dev): + raise TakeoverError(f"base bdev {base} ({path}) not created on {sid}") + created.append(("base", base)) + info = rpc.bdev_get(base) + if not info: + raise TakeoverError(f"base bdev {base} not found on {sid} after create") + size = int(info.get("block_size", 0)) * int(info.get("num_blocks", 0)) + if size < int(dev.size or 0) and ("base", base) not in created: + # A base bdev this node kept from before a LUN growth + # (device_controller.grow_cluster_devices) still has the old size: + # re-read the block device before giving up. + # The host's kernel may not have seen the growth either (the + # node was out while it ran). + try: + standby.client(timeout=_remaining(deadline, 30), retry=1).rescan_block_device(path) + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover: block rescan of %s on %s failed: %s", path, sid, e) + try: + utils.rescan_lblk_bdev(rpc, base, utils.lblk_io_engine_for({"path": path})) + info = rpc.bdev_get(base) or info + size = int(info.get("block_size", 0)) * int(info.get("num_blocks", 0)) + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover: rescan of %s on %s failed: %s", base, sid, e) + if size < int(dev.size or 0): + raise TakeoverError(f"base bdev {base} on {sid} is {size} bytes, the device {dev.size}") + + alceml = dev.alceml_bdev or f"alceml_{dev.get_id()}" + kwargs = multi_attach.alceml_create_kwargs(dev, standby, cluster) + exists = rpc.bdev_get(alceml) + warm = False + if (exists and allow_warm and kwargs.get("reservation_key") + and multi_attach.warm_standby_enabled(cluster, dev)): + warm, exists = _reuse_warm_standby(rpc, standby, alceml) + if not exists: + ok = standby.create_alceml(alceml, base, dev.get_id(), + **_alceml_open_kwargs(cluster, dev, kwargs)) + if not ok: + raise TakeoverError(f"alceml {alceml} not opened on {sid}") + created.append(("alceml", alceml)) + if kwargs.get("reservation_key") and not warm: + # Opened active with a key: it preempts the old holder (fence). + _wait_active(standby.rpc_client(timeout=1, retry=0), alceml, deadline) + try: + from simplyblock_core.controllers import qos_controller + rpc.alceml_set_qos_weights(qos_controller.get_qos_weights_list(cluster.get_id())) + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover: QoS weights not set on %s: %s", sid, e) + + pt = dev.pt_bdev or f"{alceml}_PT" + pt_info = rpc.bdev_get(pt) + if not pt_info: + if not rpc.bdev_PT_NoExcl_create(pt, alceml): + raise TakeoverError(f"PT bdev {pt} not created on {sid}") + created.append(("pt", pt)) + pt_info = rpc.bdev_get(pt) + if not pt_info: + raise TakeoverError(f"PT bdev {pt} not found on {sid} after create") + + nqn = multi_attach.device_subsystem_nqn(standby, dev, cluster) + subsys = rpc.subsystem_get(nqn) + if not subsys: + if not rpc.subsystem_create(nqn, 'sbcli-cn', dev.get_id(), + **multi_attach.subsystem_create_kwargs(standby, dev, db)): + raise TakeoverError(f"subsystem {nqn} not created on {sid}") + created.append(("subsystem", nqn)) + subsys = {} + listening = {(la or {}).get("traddr") for la in (subsys.get("listen_addresses") or [])} + ips = [] + for iface in standby.data_nics: + if not iface.ip4_address: + continue + if iface.ip4_address not in listening: + rpc.listeners_create(nqn, iface.trtype, iface.listen_address, standby.nvmf_port) + ips.append(iface.ip4_address) + if not ips: + raise TakeoverError(f"node {sid} has no data IP to export the LUN on") + if not any(ns.get("name") == pt for ns in (subsys.get("namespaces") or [])): + if not rpc.nvmf_subsystem_add_ns(nqn, pt, multi_attach.namespace_uuid(dev), + nguid=multi_attach.namespace_nguid(dev)): + raise TakeoverError(f"namespace {pt} not added to {nqn} on {sid}") + except StandbyUnreachable: + logger.warning("LUN takeover cleanup on %s skipped: SPDK not answering (%s)", + sid, [name for _, name in created]) + raise + except Exception as e: + if created and not _reachable(standby): + # Its SPDK is gone: what it held went with it (each call would + # only wait out the retries). + logger.warning("LUN takeover cleanup on %s skipped: SPDK not answering (%s)", + sid, [name for _, name in created]) + raise StandbyUnreachable(f"standby {sid}: {e}") from e + _teardown(rpc, sid, created) + raise + aliases = (pt_info or {}).get("aliases") or [] + return { + "nqn": nqn, + "ips": ips, + "port": standby.nvmf_port, + "base": base, + "pt_uuid": aliases[0] if aliases else "", + # A reused warm standby: activates on the first I/O, after the paths. + "warm": warm, + "alceml": alceml, + "pt": pt, + } + + +def _teardown(rpc, sid, created): + """Undo what this attempt created on the standby, newest first, so a + failed takeover leaves no alceml open on a LUN the node does not serve. + Never formats anything; best-effort.""" + for kind, name in reversed(created): + try: + if kind == "subsystem": + rpc.subsystem_delete(name) + elif kind == "pt": + rpc.bdev_PT_NoExcl_delete(name) + elif kind == "alceml": + rpc.bdev_alceml_delete(name) + elif kind == "base": + engine = utils.bdev_io_engine(rpc.bdev_get(name)) or constants.LBLK_IO_ENGINE_AIO + utils.delete_lblk_bdev(rpc, name, engine) + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover cleanup on %s: %s %s not removed: %s", sid, kind, name, e) + + +def _activate_warm_standby(db, cluster, owner, dev, standby, failed, path, export, deadline) -> bool: + """After the paths were switched to a reused warm standby: wait until the + peers' I/O has activated it. If it does not, fence the failed node at the + array, tear the export down and redo it cold (alceml opened active with + the key, as without warm standby); the NQN and IPs stay, so the peers' + paths already point there. Returns False (device set unavailable, event + raised) only when the cold export failed as well.""" + sid, device_id, alceml = standby.get_id(), dev.get_id(), export["alceml"] + try: + _wait_active(standby.rpc_client(timeout=1, retry=0), alceml, deadline) + try: + from simplyblock_core.controllers import qos_controller + standby.rpc_client(timeout=_remaining(deadline, 10)).alceml_set_qos_weights( + qos_controller.get_qos_weights_list(cluster.get_id())) + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover: QoS weights not set on %s: %s", sid, e) + return True + except TakeoverError as e: + msg = (f"LUN takeover of device {device_id}: warm standby {alceml} on {sid} did not " + f"activate ({e}); fencing node {failed.get_id()} at the array and opening the LUN cold") + logger.warning(msg) + _event(owner.cluster_id, dev, msg, EventObj.LEVEL_WARN) + try: + _fence(cluster, dev, failed) + _record_fenced(db, owner, device_id, failed.get_id()) + rpc = standby.rpc_client(timeout=_remaining(deadline, 60)) + _teardown(rpc, sid, [("alceml", alceml), ("pt", export["pt"]), ("subsystem", export["nqn"])]) + if rpc.bdev_get(alceml): + raise TakeoverError(f"standby alceml {alceml} not closed on {sid}") + _export_on_standby(db, cluster, standby, dev, path, deadline, allow_warm=False) + logger.info("LUN takeover of %s: opened cold on %s after the warm standby failed", + device_id, sid) + return True + except Exception as e: # noqa: BLE001 + msg = (f"LUN takeover of device {device_id} from node {failed.get_id()} failed: the warm " + f"standby on {sid} did not activate and the cold open failed: {e}") + logger.error(msg) + _event(owner.cluster_id, dev, msg, EventObj.LEVEL_CRITICAL) + # The record names the standby, so the takeover worker would not mark + # the device lost: do it here (the regular node-loss behaviour). + try: + from simplyblock_core.controllers import device_controller + device_controller.device_set_unavailable(device_id) + except Exception as e2: # noqa: BLE001 + logger.error("LUN takeover of %s: device not set unavailable: %s", device_id, e2) + return False + + +# --------------------------------------------------------------- 5. record + +def _record_serving(db, owner, device_id, failed_node_id, standby, path, export) -> bool: + standby_id = standby.get_id() + ips = export["ips"] + + def _mut(n): + d = _device_on_owner(n, device_id) + if d is None or d.serving_node() not in (failed_node_id, standby_id): + return False + d.serving_node_id = "" if standby_id == d.node_id else standby_id + paths = dict(d.node_paths or {}) + paths[standby_id] = {"path": path, "bdev": export["base"]} + d.node_paths = paths + d.nvmf_nqn = export["nqn"] + d.nvmf_ip = ",".join(ips) + d.nvmf_port = export["port"] + d.nvmf_multipath = len(ips) > 1 + if export["pt_uuid"]: + d.pt_spdk_uuid = export["pt_uuid"] + return True + + fresh = db.atomic_update(owner, _mut) + d = _device_on_owner(fresh, device_id) if fresh is not None else None + return d is not None and d.serving_node() == standby_id + + +# ---------------------------------------------------------- 6 + 7. paths + +def _failed_paths(failed, old_ips, old_port): + """``(ips, port, trtype)`` of the failed node's export.""" + if failed is None: + return [], None, "TCP" + ips = list(old_ips or []) + for iface in failed.data_nics or []: + if iface.ip4_address and iface.ip4_address not in ips: + ips.append(iface.ip4_address) + return ips, old_port or failed.nvmf_port, _transport(failed) or "TCP" + + +def _switch_paths(db, cluster_id, dev, standby, failed, failed_paths) -> list: + """Add the standby's export as a path of ``remote_alceml_`` on every + ONLINE node, then detach the failed node's path there; on all nodes in + parallel. Returns the ids of the nodes that have the new path.""" + name = f"remote_{dev.alceml_bdev}" + trtype = _transport(standby) or "TCP" + new_ips = [ip.strip() for ip in (dev.nvmf_ip or "").split(",") if ip.strip()] + dead_ips, dead_port, dead_trtype = failed_paths + dead_ips = [ip for ip in dead_ips if ip not in new_ips] + failed_id = failed.get_id() if failed is not None else None + nodes = [n for n in db.get_storage_nodes_by_cluster_id(cluster_id) + if n.status == StorageNode.STATUS_ONLINE and n.get_id() != failed_id] + + def _one(node): + try: + rpc = node.rpc_client(timeout=5, retry=1) + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover: no RPC client for %s: %s", node.get_id(), e) + return False + ok = False + for ip in new_ips: + try: + if rpc.bdev_nvme_attach_controller( + name, dev.nvmf_nqn, ip, dev.nvmf_port, trtype, multipath="multipath", + **multi_attach.attach_controller_kwargs(dev)): + ok = True + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover: path %s of %s not added on %s: %s", + ip, name, node.get_id(), e) + if not ok: + logger.error("LUN takeover: %s on %s has no path to the new export; the health " + "check reconnects it from the record", name, node.get_id()) + return False + try: + rpc.bdev_nvme_set_multipath_policy(f"{name}n1", "active_passive") + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover: multipath policy of %sn1 on %s not set: %s", + name, node.get_id(), e) + for ip in dead_ips: + try: + rpc.bdev_nvme_detach_controller(name, traddr=ip, trsvcid=dead_port, + trtype=dead_trtype) + except Exception as e: # noqa: BLE001 + # Usually: the path is already gone (the hold destructed it). + logger.info("LUN takeover: old path %s of %s on %s not detached: %s", + ip, name, node.get_id(), e) + # The node's distribs may have been told the device is unavailable + # while its old path failed; it is served again. On 1+0 a device left + # unavailable in one node's map fails every read there (2026-11-21 + # FTT2 bed: an LVS failover read on such a node aborted its SPDK). + try: + from simplyblock_core import distr_controller + distr_controller.send_dev_status_event(dev, NVMeDevice.STATUS_ONLINE, node) + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover: device status of %s not sent to %s: %s", + dev.get_id(), node.get_id(), e) + return True + + if not nodes: + return [] + # All nodes at once: the held I/O of each resumes when its path is added. + with ThreadPoolExecutor(max_workers=min(16, len(nodes)), + thread_name_prefix=f"paths-{dev.get_id()[:8]}") as pool: + results = list(pool.map(_one, nodes)) + return [n.get_id() for n, ok in zip(nodes, results) if ok] + + +def retire_stale_local_stack(node) -> list: + """Retire, on a node that is back ONLINE, what it still holds of the LUNs + another node took over while it was unreachable: its own path in each + ``remote_alceml_`` controller (the loopback to its now-stale export), + then that export's subsystem, PT bdev and alceml. Nothing is formatted; the + base bdev over the LUN is left alone (the node is fenced from the LUN). + + Needed when the node returns without a restart (a management-plane blip + long enough to start a takeover): a restart never builds these stacks + (``multi_attach.served_elsewhere``), but a node that never restarted still + has them, and its own distribs would keep writing through its loopback + into an alceml whose LUN the array has fenced. Returns the ids of the + devices retired; best-effort and idempotent.""" + from simplyblock_core.db_controller import DBController + db = DBController() + node_id = node.get_id() + try: + rpc = node.rpc_client(timeout=10, retry=1) + except Exception as e: # noqa: BLE001 + logger.warning("LUN retire: no RPC client for %s: %s", node_id, e) + return [] + retired = [] + for owner in db.get_storage_nodes_by_cluster_id(node.cluster_id): + for dev in owner.nvme_devices or []: + if not multi_attach.is_multi_attach(dev) or not multi_attach.served_elsewhere(dev, node_id): + continue + # Only a node that once exported the LUN can hold a stale stack: + # its owner, or a node it was taken over by before. + if node_id != dev.node_id and node_id not in (dev.node_paths or {}): + continue + pt = f"{dev.alceml_bdev}_PT" + try: + has_stack = bool(rpc.bdev_get(dev.alceml_bdev)) or bool(rpc.bdev_get(pt)) + except Exception: # noqa: BLE001 + has_stack = False + if not has_stack: + continue + if _is_kept_warm_standby(db, rpc, node, dev, pt): + continue + name = f"remote_{dev.alceml_bdev}" + own_ips = [iface.ip4_address for iface in node.data_nics or [] if iface.ip4_address] + for ip in own_ips: + try: + rpc.bdev_nvme_detach_controller(name, traddr=ip, trsvcid=node.nvmf_port, + trtype=_transport(node) or "TCP") + except Exception as e: # noqa: BLE001 + logger.info("LUN retire: own path %s of %s on %s not detached: %s", + ip, name, node_id, e) + nqn = multi_attach.device_subsystem_nqn(node, dev, db.get_cluster_by_id(node.cluster_id)) + _teardown(rpc, node_id, [("alceml", dev.alceml_bdev), ("pt", pt), ("subsystem", nqn)]) + logger.warning("LUN retire: %s on %s is served by %s since a takeover; its stale " + "local stack was removed (nothing formatted)", + dev.get_id(), node_id, dev.serving_node()) + retired.append(dev.get_id()) + return retired + + +def _is_kept_warm_standby(db, rpc, node, dev, pt) -> bool: + """Whether what ``node`` holds of ``dev`` is only the warm standby + ensure_warm_standbys keeps there (an unexported instance opened standby) + on a node not fenced from the LUN: not a stale stack. Always False + without warm standby.""" + if multi_attach.fenced_from(dev, node.get_id()): + return False + try: + cluster = db.get_cluster_by_id(node.cluster_id) + if not multi_attach.warm_standby_enabled(cluster, dev) or rpc.bdev_get(pt): + return False + except Exception: # noqa: BLE001 + return False + role = _alceml_role(node, dev.alceml_bdev) + return _is_standby_instance(role) and role.get("standby_at_open") is True + + +def reintegrate_fenced_node(node) -> list: + """Make a node that is back ONLINE a standby again for every LUN a + takeover fenced it from: unfence it at the array (re-present the LUN). + + Only when nothing it held can still reach the LUN late: + + * the LUN is served by another ONLINE node (the takeover completed; a + fenced node of a takeover still holding is never unfenced); + * the node holds no alceml or PT bdev of the device (its stale stack was + retired, or a restart never built it); + * the node's host shows no disk carrying the LUN (``lun_info`` empty): the + fence removed every path, so no host-side queue (multipath + ``queue_if_no_path`` included) holds writes for it. A host that still + shows the LUN keeps the fence until an operator calls + :func:`unfence_node`. + + Returns the ids of the devices unfenced; best-effort, idempotent.""" + db = DBController() + node_id = node.get_id() + if node.status != StorageNode.STATUS_ONLINE: + return [] + nodes = db.get_storage_nodes_by_cluster_id(node.cluster_id) + by_id = {n.get_id(): n for n in nodes} + todo = [dev for owner in nodes for dev in owner.nvme_devices or [] + if multi_attach.fenced_from(dev, node_id)] + if not todo: + return [] + try: + rpc = node.rpc_client(timeout=10, retry=1) + except Exception as e: # noqa: BLE001 + logger.info("LUN reintegrate: no RPC client for %s: %s", node_id, e) + return [] + done = [] + for dev in todo: + serving = by_id.get(dev.serving_node()) + if serving is None or serving.get_id() == node_id or serving.status != StorageNode.STATUS_ONLINE: + continue + if fence_in_flight(dev.get_id(), node_id): + # Its detach is still running: an unfence now is undone by it + # (2026-11-20 resize bed, R3: the node stayed detached for good + # while the record listed it as a standby). + logger.info("LUN reintegrate: array fence of %s on %s still running; stays fenced", + node_id, dev.lun_wwn) + continue + stale = _holds_stack(rpc, dev) + if stale is None: + # SPDK not answering (a node just lost may still read ONLINE): + # nothing proves it holds no stack. + logger.info("LUN reintegrate: SPDK of %s not answering; stays fenced from %s", + node_id, dev.lun_wwn) + continue + if stale: + retire_stale_local_stack(node) + stale = _holds_stack(rpc, dev) + if stale is None: + stale = True + if stale: + logger.info("LUN reintegrate: %s still holds a stack of %s; stays fenced", + node_id, dev.get_id()) + continue + try: + result, err = node.client(timeout=10, retry=0, connect_retry=0).lun_info(dev.lun_wwn) + except Exception as e: # noqa: BLE001 + logger.info("LUN reintegrate: agent of %s not reached: %s", node_id, e) + continue + if err or not isinstance(result, list): + logger.info("LUN reintegrate: LUN inventory of %s unavailable (%s)", node_id, err) + continue + if result: + logger.warning("LUN reintegrate: host of %s still shows LUN %s (%s); it stays fenced " + "until an operator unfences it", node_id, dev.lun_wwn, + [e.get("device_path") for e in result if isinstance(e, dict)]) + continue + if unfence_node(dev.get_id(), node_id): + logger.info("LUN reintegrate: %s is a standby of LUN %s again", node_id, dev.lun_wwn) + done.append(dev.get_id()) + return done + + +def _holds_stack(rpc, dev): + """Whether the node holds an alceml or PT bdev of ``dev``; None when its + SPDK does not answer.""" + try: + return bool(rpc.bdev_get(dev.alceml_bdev)) or bool(rpc.bdev_get(f"{dev.alceml_bdev}_PT")) + except Exception: # noqa: BLE001 + return None + + +# ------------------------------------------------------------- operator + +def unfence_node(device_id, node_id) -> bool: + """Give ``node_id`` access to the LUN of ``device_id`` again and drop it + from ``fenced_node_ids``. Called by :func:`reintegrate_fenced_node` once + the node's host shows no disk of the LUN, else by an operator once the + host is known to hold no queued writes for it (its multipath queues + drained or the host rebooted). Returns True on success.""" + db = DBController() + try: + dev = db.get_storage_device_by_id(device_id) + owner = db.get_storage_node_by_id(dev.node_id) + node = db.get_storage_node_by_id(node_id) + cluster = db.get_cluster_by_id(owner.cluster_id) + except KeyError as e: + logger.error("Unfence of %s on %s: %s", node_id, device_id, e) + return False + if dev.serving_node() == node_id: + logger.error("Unfence of %s on %s refused: it is the serving node", node_id, device_id) + return False + + # The record first, the array second. The attachment reconcile + # (array_controller.reconcile_lun_attachments, health check) detaches + # every host the record still lists as fenced: attaching first left a + # window in which it detached the LUN again (2026-11-19 FTT1 bed, kill of + # an owner's SPDK). Once the record is clear the reconcile attaches the + # node itself, so a failed array call below is retried there. + def _mut(n): + d = _device_on_owner(n, device_id) + if d is None or node_id not in d.fenced_node_ids: + return False + d.fenced_node_ids = [x for x in d.fenced_node_ids if x != node_id] + return True + + db.atomic_update(owner, _mut) + _event(owner.cluster_id, dev, f"Node {node_id} unfenced from LUN {dev.lun_wwn} of device {device_id}") + try: + if not array.plugin_for_cluster(cluster).unfence( + multi_attach.array_lun_ref(dev), multi_attach.array_host_ref(node)): + logger.warning("Unfence of %s on LUN %s not acknowledged by the array; the " + "attachment reconcile presents it", node_id, dev.lun_wwn) + except Exception as e: # noqa: BLE001 + logger.warning("Array unfence of %s on LUN %s failed (%s); the attachment " + "reconcile presents it", node_id, dev.lun_wwn, e) + return True + + +# ---------------------------------------------------------- warm standby + +# node id -> the device ids whose warm standby did not fit its memory budget +# when last reported (an event is raised when the set changes). +_warm_over_budget: dict[str, frozenset] = {} + + +def ensure_warm_standbys(node) -> dict: + """Keep the warm standby alcemls of an ONLINE node in line with the + record (``Cluster.multi_attach_warm_standby`` clusters only). + + For every LUN the node is a standby of (multi_attach.warm_standby_devices: + attached, not serving, not fenced, the device ONLINE), open a standby + alceml (``standby=true``, open-only, the reservation key, the same + parameters as a takeover) over the node's own path to the LUN, unless one + is open already. It is never exported: a peer's I/O would activate it. + + Close the standby alceml (and, on a node that does not own the device, + its base bdev) of every LUN the node is no longer a standby of (fenced, + detached, removed). Never touches the serving node's stack, an exported + alceml (a PT bdev over it: retire_stale_local_stack handles those) or an + active instance, nor a device whose takeover runs in this process. + + A standby holds the LUN's page map in the node's SPDK memory: one is not + opened when the node's budget (StorageNode.minimum_sys_memory, its + container limit) does not cover its own devices plus its standbys + (utils.sys_memory_for_capacity); a cluster event says so instead. A + restart sizes the budget for them (storage_node_ops). + + Returns ``{"created": [...], "removed": [...], "over_budget": [...]}`` + (device ids); best-effort and idempotent, never raises.""" + report: dict = {"created": [], "removed": [], "over_budget": []} + try: + _ensure_warm_standbys(node, report) + except Exception as e: # noqa: BLE001 + logger.error("Warm standbys of %s not reconciled: %s", node.get_id(), e) + return report + + +def _ensure_warm_standbys(node, report): + node_id = node.get_id() + db = DBController() + cluster = db.get_cluster_by_id(node.cluster_id) + if not (multi_attach.is_multi_attach(cluster) + and getattr(cluster, "multi_attach_warm_standby", False) is True): + return + nodes = db.get_storage_nodes_by_cluster_id(node.cluster_id) + node = next((n for n in nodes if n.get_id() == node_id), node) + if node.status != StorageNode.STATUS_ONLINE: + return + keep = {dev.get_id() for _, dev in multi_attach.warm_standby_devices( + cluster, node_id, nodes, multi_attach._WARM_STANDBY_BUDGET_STATES)} + want = multi_attach.warm_standby_devices(cluster, node_id, nodes) + try: + rpc = node.rpc_client(timeout=10, retry=1) + except Exception as e: # noqa: BLE001 + logger.info("Warm standbys: no RPC client for %s: %s", node_id, e) + return + + # Close what the node is no longer a standby of. + for owner in nodes: + for dev in owner.nvme_devices or []: + if (not multi_attach.is_multi_attach(dev) or dev.get_id() in keep + or dev.serving_node() == node_id or _local_lock(dev.get_id()).locked()): + continue + if _remove_warm_standby(rpc, node, dev): + report["removed"].append(dev.get_id()) + + # Open what is missing, within the node's memory budget. + committed = sum(max(int(d.size or 0), 0) for d in node.nvme_devices or []) + committed += sum(max(int(d.size or 0), 0) for o, d in multi_attach.devices_served_by(nodes, node_id) + if o.get_id() != node_id) + missing = [] + for _owner, dev in want: + try: + held = bool(rpc.bdev_get(dev.alceml_bdev)) + except Exception as e: # noqa: BLE001 + logger.info("Warm standbys: SPDK of %s not answering: %s", node_id, e) + return + if held: + committed += max(int(dev.size or 0), 0) + else: + missing.append(dev) + budget = int(getattr(node, "minimum_sys_memory", 0) or 0) + for dev in missing: + if _local_lock(dev.get_id()).locked(): + continue + size = max(int(dev.size or 0), 0) + if budget and utils.sys_memory_for_capacity(committed + size) > budget: + report["over_budget"].append(dev.get_id()) + continue + if _create_warm_standby(rpc, cluster, node, dev): + report["created"].append(dev.get_id()) + committed += size + _report_over_budget(cluster, node, report["over_budget"], committed, budget) + + +def _create_warm_standby(rpc, cluster, node, dev) -> bool: + """Open the standby alceml of ``dev`` on ``node``; False (logged) when the + node cannot read the LUN or SPDK refuses it. Closes what it opened on + failure.""" + node_id = node.get_id() + kwargs = multi_attach.alceml_create_kwargs(dev, node, cluster) + if not kwargs.get("reservation_key") or getattr(dev, "bdev_type", "aio") != "aio": + return False + path, why = _probe(node, dev, time.monotonic() + 15) + if not path: + logger.info("Warm standby of %s on %s not opened: %s", dev.get_id(), node_id, why) + return False + base = utils.aio_bdev_name_for_serial(dev.serial_number) + created = [] + try: + if not rpc.bdev_get(base): + _create_base_bdev(rpc, base, path, dev) + created.append(("base", base)) + info = rpc.bdev_get(base) or {} + size = int(info.get("block_size", 0)) * int(info.get("num_blocks", 0)) + if size < int(dev.size or 0): + raise TakeoverError(f"base bdev {base} is {size} bytes, the device {dev.size}") + ok = node.create_alceml(dev.alceml_bdev, base, dev.get_id(), + standby=True, + refresh_interval_ms=constants.MULTI_ATTACH_WARM_REFRESH_MS, + **_alceml_open_kwargs(cluster, dev, kwargs)) + if not ok: + raise TakeoverError(f"standby alceml {dev.alceml_bdev} not opened") + created.append(("alceml", dev.alceml_bdev)) + except Exception as e: # noqa: BLE001 + logger.warning("Warm standby of %s on %s not opened: %s", dev.get_id(), node_id, e) + _teardown(rpc, node_id, created) + return False + logger.info("Warm standby of %s (LUN %s) opened on %s at %s", + dev.get_id(), dev.lun_wwn, node_id, path) + return True + + +def _remove_warm_standby(rpc, node, dev) -> bool: + """Close a standby alceml of ``dev`` the node holds although it is no + longer a standby of the LUN; its base bdev too unless the node owns the + device. Only an unexported standby instance is closed.""" + alceml = dev.alceml_bdev + try: + if not alceml or not rpc.bdev_get(alceml): + return False + if rpc.bdev_get(dev.pt_bdev or f"{alceml}_PT"): + return False + except Exception: # noqa: BLE001 + return False + if _alceml_role(node, alceml).get("role") != "standby": + return False + todo = [("alceml", alceml)] + if dev.node_id != node.get_id(): + todo.insert(0, ("base", utils.aio_bdev_name_for_serial(dev.serial_number))) + _teardown(rpc, node.get_id(), todo) + logger.info("Warm standby of %s closed on %s: no longer a standby of LUN %s", + dev.get_id(), node.get_id(), dev.lun_wwn) + return True + + +def _report_over_budget(cluster, node, device_ids, committed, budget): + """A cluster event (once per change) when warm standbys of the node did + not fit its memory budget.""" + node_id = node.get_id() + now = frozenset(device_ids) + if _warm_over_budget.get(node_id, frozenset()) == now: + return + _warm_over_budget[node_id] = now + if not now: + return + msg = (f"Node {node_id}: warm standbys of {len(now)} LUN(s) ({', '.join(sorted(now))}) not " + f"opened: its SPDK memory budget {utils.humanbytes(budget)} covers its own devices and " + f"standbys of {utils.humanbytes(committed)} only. Restart the node to size its memory " + f"for its standbys; a takeover of these LUNs opens them cold meanwhile.") + logger.warning(msg) + try: + ec.log_event_cluster( + cluster_id=cluster.get_id(), + domain=ec.DOMAIN_CLUSTER, + event=ec.EVENT_LUN_TAKEOVER, + db_object=node, + caused_by=ec.CAUSED_BY_MONITOR, + message=msg, + node_id=node_id, + event_level=EventObj.LEVEL_WARN) + except Exception as e: # noqa: BLE001 + logger.error("Warm standby budget event of %s not logged: %s", node_id, e) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 29daecb3b6..3e6a1bf864 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -7,7 +7,7 @@ from datetime import datetime from typing import Any -from simplyblock_core import constants, index_ops, utils +from simplyblock_core import constants, floors, index_ops, utils from simplyblock_core.controllers import ( activation_gate, events_controller, @@ -1044,11 +1044,17 @@ def add_lvol_ha(name, size, host_id_or_name, ha_type, pool_id_or_name, use_comp= logger.error(mgs) return False, mgs - if ndcs or npcs: - if ndcs+npcs > len(online_nodes): - mgs = f"Online storage nodes: {len(online_nodes)} are less than the required LVol geometry: {(ndcs+npcs)}" - logger.error(mgs) - return False, mgs + # Resource floors (simplyblock_core.floors) for the volume's effective + # scheme -- its own ndcs/npcs when given, the cluster's otherwise -- + # judged on the ONLINE nodes, their data devices and (when used) failure + # domains. The refusal names the violated floor. + floor_violations = floors.evaluate(cl, snodes, ndcs=ndcs, npcs=npcs, online_nodes=online_nodes) + if floor_violations: + eff_ndcs, eff_npcs = floors.scheme(cl, ndcs, npcs) + mgs = (f"Cannot create the volume: the cluster is below the resource floors for the " + f"{eff_ndcs}+{eff_npcs} scheme: " + "; ".join(floor_violations)) + logger.error(mgs) + return False, mgs if cluster_size_total <= 0: # Every online device reported zero usable capacity: no meaningful diff --git a/simplyblock_core/controllers/storage_events.py b/simplyblock_core/controllers/storage_events.py index e83ddbb696..30a2aa4125 100644 --- a/simplyblock_core/controllers/storage_events.py +++ b/simplyblock_core/controllers/storage_events.py @@ -144,6 +144,20 @@ def snode_rpc_timeout(node, timeout_seconds, caused_by=ec.CAUSED_BY_MONITOR): node_id=node.get_id()) +def snode_port_check_correlated(node, reason, caused_by=ec.CAUSED_BY_MONITOR): + """Multi-attach clusters: more nodes than the cluster tolerates fail only + their port check at once; none of them is set DOWN for it.""" + ec.log_event_cluster( + cluster_id=node.cluster_id, + domain=ec.DOMAIN_CLUSTER, + event=ec.EVENT_STATUS_CHANGE, + db_object=node, + caused_by=caused_by, + event_level=EventObj.LEVEL_ERROR, + message=f"Port checks failing cluster-wide, nodes not set down: {reason}", + node_id=node.get_id()) + + def snode_orphaned_objects(node, orphans, caused_by=ec.CAUSED_BY_MONITOR): """Raised by the lvol monitor's orphan sweep: lvstore objects exist on this node with no record in the database. Detect-only — see diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index e9c7dad063..5936dcecce 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -8,7 +8,13 @@ from collections.abc import Callable from typing import NamedTuple -from simplyblock_core import constants, db_controller, storage_node_ops, utils +from simplyblock_core import ( + constants, + db_controller, + multi_attach, + storage_node_ops, + utils, +) from simplyblock_core.controllers import ( activation_gate, device_controller, @@ -848,7 +854,13 @@ def add_node_to_auto_restart(node): for sn in db.get_storage_nodes_by_cluster_id(node.cluster_id): if node.get_id() != sn.get_id() and sn.status != StorageNode.STATUS_ONLINE and node.mgmt_ip != sn.mgmt_ip: offline_nodes += 1 - if offline_nodes > cluster.distr_npcs and cluster.status != Cluster.STATUS_SUSPENDED: + # A multi-attach cluster runs 1+0 (npcs 0) and tolerates FTT node + # losses through LUN takeovers: with npcs, two nodes down at FTT 2 + # blocked each other's restart for good while the cluster stayed + # DEGRADED (2026-11-20 FTT2 bed, network cut + SPDK kill). + tolerated = (cluster.max_fault_tolerance if multi_attach.is_multi_attach(cluster) + else cluster.distr_npcs) + if offline_nodes > tolerated and cluster.status != Cluster.STATUS_SUSPENDED: logger.info("Node found that is not online, skip node auto restart") return False return _add_task(JobSchedule.FN_NODE_RESTART, node.cluster_id, node.get_id(), "", max_retry=11) diff --git a/simplyblock_core/credentials.py b/simplyblock_core/credentials.py new file mode 100644 index 0000000000..070220625c --- /dev/null +++ b/simplyblock_core/credentials.py @@ -0,0 +1,316 @@ +"""Credentials for external systems (storage arrays, ...), kept out of the database. + +Plugins that drive an external system (provision/resize/delete a LUN on a +storage array, ...) never store or fetch secrets themselves. Their records +hold a :data:`CredentialRef`; at call time they ask a +:class:`CredentialProvider` to resolve it into a :class:`Credential`, which +exists only in memory. + +Two kinds of reference: + +*Managed* (:class:`ManagedCredentialRef`) + simplyblock received the secret (CLI/API) and keeps it wrapped under a + cluster KEK in the cluster's KMS: Vault Transit or a KMIP appliance. Only + the ciphertext is stored, inside the ref. Revoking the KEK in the KMS + makes every managed credential of the cluster unreadable at once. + +*External* (:class:`VaultKVCredentialRef`, :class:`KMIPSecretCredentialRef`) + The customer owns and rotates the secret in their own Vault KV v2 mount or + as a KMIP ``SecretData`` object; simplyblock stores only the pointer. + +Every resolution is bound to a *context* string naming what the credential is +for (e.g. ``"array//https://pure01.example:443"``). A managed credential +carries its context inside the wrapped payload and is refused for any other, +so rewriting an array's endpoint in the database cannot redirect the +credential to another host. An external secret may opt into the same check by +holding a ``_context`` field. + +With no external KMS configured the cluster falls back to ``LocalKMS``, which +only encodes. Managed credentials are then refused unless the cluster has +explicitly opted in via ``Cluster.allow_local_kms_credentials``. + +Resolved credentials are cached in memory for ``ttl_s``. On an authentication +failure from the external system the caller invalidates the ref and resolves +again once, which picks up a rotated external secret without a restart. +""" +from __future__ import annotations + +import json +import logging +import threading +import time +from collections.abc import Callable, Mapping +from typing import TYPE_CHECKING, Annotated, Literal, Union + +from pydantic import BaseModel, ConfigDict, Field, SecretStr, TypeAdapter + +from simplyblock_core.kms import ( + KMS, + HCPClient, + KMIPClient, + KMIPKMSSettings, + KMSException, + VaultKMSSettings, +) + +if TYPE_CHECKING: + from simplyblock_core.models.cluster import Cluster + +logger = logging.getLogger(__name__) + +#: Field an external secret may hold to bind itself to one context. +CONTEXT_FIELD = "_context" + +Backend = Literal["vault", "kmip", "local"] + + +class CredentialError(Exception): + """A credential could not be stored or resolved. Never carries secret values.""" + + +class ManagedCredentialRef(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: Literal["managed"] = "managed" + #: The KMS backend that wrapped it; a different backend cannot unwrap it. + backend: Backend + kek_name: str = Field(min_length=1) + wrapped: str = Field(min_length=1) + + +class VaultKVCredentialRef(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: Literal["vault-kv"] = "vault-kv" + mount: str = Field(min_length=1) + path: str = Field(min_length=1) + #: Pin a KV version; ``None`` follows the latest (i.e. picks up rotation). + version: int | None = Field(default=None, ge=1) + + +class KMIPSecretCredentialRef(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: Literal["kmip-secret"] = "kmip-secret" + uid: str = Field(min_length=1) + #: ``None``: the object's value is a JSON object of string fields. + #: Set: the raw value (UTF-8) is that single field, e.g. ``"password"``. + field: str | None = Field(default=None, min_length=1) + + +CredentialRef = Annotated[ + Union[ManagedCredentialRef, VaultKVCredentialRef, KMIPSecretCredentialRef], + Field(discriminator="kind"), +] +CREDENTIAL_REF_ADAPTER: TypeAdapter[CredentialRef] = TypeAdapter(CredentialRef) + + +class Credential: + """Resolved secret fields, in memory only. ``repr`` shows field names only.""" + + __slots__ = ("_fields",) + + def __init__(self, fields: Mapping[str, SecretStr]): + self._fields = dict(fields) + + @property + def field_names(self) -> frozenset[str]: + return frozenset(self._fields) + + def get(self, name: str) -> SecretStr: + try: + return self._fields[name] + except KeyError: + raise CredentialError( + f"Credential has no field {name!r} (has: {', '.join(sorted(self._fields))})" + ) from None + + def get_optional(self, name: str) -> SecretStr | None: + return self._fields.get(name) + + def __repr__(self) -> str: + return f"Credential(fields={sorted(self._fields)})" + + +def kek_name_for(cluster_id: str) -> str: + """The cluster KEK that wraps managed credentials.""" + return f"credentials-{cluster_id}" + + +def _backend_of(cluster: Cluster) -> Backend: + settings = cluster.get_kms_settings() + if settings is None: + return "local" + if isinstance(settings, VaultKMSSettings): + return "vault" + if isinstance(settings, KMIPKMSSettings): + return "kmip" + raise CredentialError(f"Unsupported KMS settings: {type(settings).__name__}") + + +class CredentialProvider: + """Stores and resolves credentials for one cluster. Thread-safe.""" + + def __init__( + self, + cluster: Cluster, + *, + ttl_s: float = 300.0, + clock: Callable[[], float] = time.monotonic, + kms_factory: Callable[..., KMS] | None = None, + ): + self._cluster = cluster + self._ttl_s = ttl_s + self._clock = clock + self._kms_factory = kms_factory + self._lock = threading.Lock() + self._cache: dict[tuple[str, str], tuple[float, Credential]] = {} + + # -- KMS access --------------------------------------------------------- + + def _open_kms(self) -> KMS: + backend = _backend_of(self._cluster) + if backend == "local" and not self._cluster.allow_local_kms_credentials: + raise CredentialError( + "No external KMS (Vault or KMIP) is configured for this cluster. " + "Configure one, or explicitly opt in to keeping credentials in the " + "cluster database (effectively unencrypted) by setting " + "allow_local_kms_credentials on the cluster." + ) + cert_role = None + settings = self._cluster.get_kms_settings() + if isinstance(settings, VaultKMSSettings): + cert_role = settings.credentials_cert_role + factory = self._kms_factory + if factory is None: + # Looked up per call, not bound at import: the KMS connection is + # replaced in tests and by callers that patch the package. + from simplyblock_core import kms as kms_pkg + factory = kms_pkg.create_kms_connection + try: + return factory(self._cluster, cert_role=cert_role) + except KMSException as e: + raise CredentialError(f"Cannot connect to the KMS: {e.message}") from e + + # -- Store -------------------------------------------------------------- + + def store( + self, fields: Mapping[str, SecretStr | str], context: str, + ) -> ManagedCredentialRef: + """Wrap ``fields`` under the cluster's credentials KEK, bound to + ``context``. Persist the returned ref; the plaintext is not kept.""" + if not context: + raise CredentialError("A credential needs a non-empty context") + if not fields: + raise CredentialError("Refusing to store an empty credential") + secret_fields = { + name: value if isinstance(value, SecretStr) else SecretStr(value) + for name, value in fields.items() + } + backend = _backend_of(self._cluster) + kek_name = kek_name_for(self._cluster.get_id()) + try: + with self._open_kms() as kms: + kms.ensure_key_encryption_key(kek_name) + wrapped = kms.wrap_secret(kek_name, secret_fields, context) + except KMSException as e: + raise CredentialError(f"Storing credential failed: {e.message}") from e + logger.info("Stored managed credential (backend=%s, %d fields, context=%s)", + backend, len(secret_fields), context) + return ManagedCredentialRef(backend=backend, kek_name=kek_name, wrapped=wrapped) + + # -- Resolve ------------------------------------------------------------ + + @staticmethod + def _cache_key(ref: CredentialRef, context: str) -> tuple[str, str]: + return ref.model_dump_json(), context + + def resolve(self, ref: CredentialRef | Mapping, context: str) -> Credential: + """Return the credential for ``ref``, bound to ``context``.""" + if isinstance(ref, Mapping): + ref = CREDENTIAL_REF_ADAPTER.validate_python(ref) + key = self._cache_key(ref, context) + now = self._clock() + with self._lock: + hit = self._cache.get(key) + if hit is not None and hit[0] > now: + return hit[1] + + credential = self._fetch(ref, context) + + with self._lock: + self._cache[key] = (self._clock() + self._ttl_s, credential) + return credential + + def invalidate(self, ref: CredentialRef | None = None) -> None: + """Drop cached resolutions of ``ref`` (all, if ``None``). Call after + the external system rejected the credential, then resolve once more.""" + with self._lock: + if ref is None: + self._cache.clear() + return + dumped = ref.model_dump_json() + for key in [k for k in self._cache if k[0] == dumped]: + del self._cache[key] + + def _fetch(self, ref: CredentialRef, context: str) -> Credential: + if isinstance(ref, ManagedCredentialRef): + backend = _backend_of(self._cluster) + if ref.backend != backend: + raise CredentialError( + f"Credential was stored with the {ref.backend} KMS backend, but the " + f"cluster now uses {backend}; store it again" + ) + try: + with self._open_kms() as kms: + if isinstance(ref, ManagedCredentialRef): + return Credential(kms.unwrap_secret(ref.kek_name, ref.wrapped, context)) + if isinstance(ref, VaultKVCredentialRef): + if not isinstance(kms, HCPClient): + raise CredentialError( + "A Vault KV credential needs the cluster KMS to be Vault") + fields = kms.read_kv_secret(ref.mount, ref.path, ref.version) + where = f"Vault {ref.mount}/{ref.path}" + else: + if not isinstance(kms, KMIPClient): + raise CredentialError( + "A KMIP secret credential needs the cluster KMS to be KMIP") + fields = self._parse_secret_data(kms.get_secret_data(ref.uid), ref) + where = f"KMIP object {ref.uid}" + except KMSException as e: + raise CredentialError(f"Resolving credential failed: {e.message}") from e + return Credential(self._check_external_context(fields, context, where)) + + @staticmethod + def _parse_secret_data( + raw: bytes, ref: KMIPSecretCredentialRef, + ) -> dict[str, SecretStr]: + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as e: + raise CredentialError(f"KMIP object {ref.uid} is not UTF-8 text") from e + if ref.field is not None: + return {ref.field: SecretStr(text)} + try: + data = json.loads(text) + except ValueError: + data = None + if not isinstance(data, dict) or not data or not all( + isinstance(k, str) and isinstance(v, str) for k, v in data.items() + ): + raise CredentialError( + f"KMIP object {ref.uid} is not a JSON object of string fields; " + "set 'field' on the reference to use its raw value as one field" + ) + return {name: SecretStr(value) for name, value in data.items()} + + @staticmethod + def _check_external_context( + fields: dict[str, SecretStr], context: str, where: str, + ) -> dict[str, SecretStr]: + bound = fields.pop(CONTEXT_FIELD, None) + if bound is not None and bound.get_secret_value() != context: + raise CredentialError(f"Secret at {where} is bound to a different context") + if not fields: + raise CredentialError(f"Secret at {where} has no credential fields") + return fields diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index d8a96c85b2..e2cf676660 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -1954,6 +1954,109 @@ def release_lvol_ns_slot(self, lvol): return transactional(self, kv, lvol) return lvol.remove(kv) + # ---- Multi-attach controller-ID block claim (Single FDB Transaction) ---- + + CNTLID_BLOCK_ALLOC_PREFIX = "cntlid_block_alloc" + + def _claim_cntlid_block_tx(self, tr, node_key, alloc_key, node_id, live_node_ids, + max_blocks, index_list): + # The per-cluster allocation map {node_id: block} is the authority: + # every claim reads and rewrites it, so two concurrent claims in one + # cluster conflict and the loser retries against the winner's entry. + raw = tr.get(alloc_key).wait() + alloc = json.loads(bytes(raw).decode()) if raw.present() else {} + if node_id in alloc: + block = int(alloc[node_id]) + else: + # Entries of nodes whose record is gone are reclaimed; a REMOVED + # node keeps its block (its record still exists). + alloc = {nid: b for nid, b in alloc.items() if nid in live_node_ids} + used = {int(b) for b in alloc.values()} + free = next((i for i in range(max_blocks) if i not in used), None) + if free is None: + raise ValueError( + f"no free controller-ID block: a multi-attach cluster supports " + f"at most {max_blocks} storage nodes") + block = free + alloc[node_id] = block + tr.set(alloc_key, json.dumps(alloc, sort_keys=True).encode()) + + def _set(node): + if node.cntlid_block == block: + return False + node.cntlid_block = block + return True + + self._atomic_update_tx(tr, node_key, StorageNode, _set, index_list) + return block + + def claim_cntlid_block(self, node, max_blocks): + """Claim the smallest free controller-ID block of ``node``'s cluster + for ``node`` (idempotent: a node that holds one gets it back) and + persist it on the node record, in ONE FDB transaction. Returns the + block index. + + Raises: + ValueError: every block of the cluster is taken. + """ + kv = self.kv_store + alloc_key = f"{self.CNTLID_BLOCK_ALLOC_PREFIX}/{node.cluster_id}".encode() + node_key = node.get_db_id().encode() + live_node_ids = {n.get_id() for n in self.get_storage_nodes_by_cluster_id(node.cluster_id)} + live_node_ids.add(node.get_id()) + index_list = StorageNode.active_indexes(kv) + if kv is not None and hasattr(kv, 'create_transaction'): + transactional = fdb.transactional(DBController._claim_cntlid_block_tx) + return transactional(self, kv, node_key, alloc_key, node.get_id(), + live_node_ids, max_blocks, index_list) + return self._claim_cntlid_block_tx( + _NoTxnStore(kv), node_key, alloc_key, node.get_id(), live_node_ids, + max_blocks, index_list) + + # ---- Multi-attach LUN takeover lease (Single FDB Transaction) ---- + + LUN_TAKEOVER_LOCK_PREFIX = "lun_takeover_lock" + + @staticmethod + def _acquire_lun_takeover_lock_tx(tr, key, owner, now, ttl): + raw = tr.get(key).wait() + if raw.present(): + held = json.loads(bytes(raw).decode()) + if (held.get("owner") and held.get("owner") != owner + and now - float(held.get("at", 0)) <= ttl): + return False, held.get("owner") + tr.set(key, json.dumps({"owner": owner, "at": now}).encode()) + return True, None + + @staticmethod + def _release_lun_takeover_lock_tx(tr, key, owner): + raw = tr.get(key).wait() + if raw.present() and json.loads(bytes(raw).decode()).get("owner") == owner: + tr.clear(key) + + def acquire_lun_takeover_lock(self, device_id, owner, ttl): + """Take the takeover lease of one multi-attach LUN for ``owner``, so + two takeovers of the same LUN never run together, in this process or + in another. A lease older than ``ttl`` seconds (its holder died) is + taken over. Returns ``(True, None)`` or ``(False, holder)``.""" + key = f"{self.LUN_TAKEOVER_LOCK_PREFIX}/{device_id}".encode() + now = time.time() + kv = self.kv_store + if kv is not None and hasattr(kv, 'create_transaction'): + transactional = fdb.transactional(DBController._acquire_lun_takeover_lock_tx) + return transactional(kv, key, owner, now, ttl) + return DBController._acquire_lun_takeover_lock_tx(_NoTxnStore(kv), key, owner, now, ttl) + + def release_lun_takeover_lock(self, device_id, owner): + """Release the lease only if ``owner`` still holds it.""" + key = f"{self.LUN_TAKEOVER_LOCK_PREFIX}/{device_id}".encode() + kv = self.kv_store + if kv is not None and hasattr(kv, 'create_transaction'): + transactional = fdb.transactional(DBController._release_lun_takeover_lock_tx) + transactional(kv, key, owner) + return + DBController._release_lun_takeover_lock_tx(_NoTxnStore(kv), key, owner) + # ---- Generic atomic read-modify-write (Single FDB Transaction) ---- def _atomic_update_tx(self, tr, key, model_cls, mutate_fn, index_list): diff --git a/simplyblock_core/distr_controller.py b/simplyblock_core/distr_controller.py index 427bcf2ffe..5e588b96e8 100644 --- a/simplyblock_core/distr_controller.py +++ b/simplyblock_core/distr_controller.py @@ -3,7 +3,7 @@ import re import threading -from simplyblock_core import utils +from simplyblock_core import multi_attach, utils from simplyblock_core.db_controller import DBController from simplyblock_core.models.nvme_device import NVMeDevice, RemoteDevice from simplyblock_core.models.storage_node import StorageNode @@ -407,10 +407,15 @@ def get_distr_cluster_map(snodes, target_node, distr_name="", home_site=None): dev_w_gib = utils.convert_size(dev.weight_size(), 'GiB') or 1 name = None dev_status = dev.status - if snode.get_id() == target_node.get_id(): - name = dev.alceml_bdev + own = snode.get_id() == target_node.get_id() + if own: local_node_index = index + if own and not multi_attach.is_multi_attach(dev): + name = dev.alceml_bdev else: + # A peer's device, or an own multi-attach device: the owner's + # distribs reach it through its loopback controller too, so a + # takeover path covers them as well. dev2 = target_remote_by_id.get(dev.get_id()) if dev2 is not None: name = dev2.remote_bdev @@ -766,7 +771,7 @@ def send_cluster_map_add_device(device: NVMeDevice, target_node: StorageNode): if target_node.status == StorageNode.STATUS_ONLINE: rpc_client = target_node.rpc_client(timeout=3) - if target_node.get_id() == dnode.get_id(): + if target_node.get_id() == dnode.get_id() and not multi_attach.is_multi_attach(device): name = device.alceml_bdev else: name = f"remote_{device.alceml_bdev}n1" diff --git a/simplyblock_core/floors.py b/simplyblock_core/floors.py new file mode 100644 index 0000000000..92c156e4b6 --- /dev/null +++ b/simplyblock_core/floors.py @@ -0,0 +1,272 @@ +"""Erasure-coding resource floors (product rules of 2026-11-09). + +A cluster with the scheme n+k (``cluster.distr_ndcs`` = n, ``cluster.distr_npcs`` += k) has five FLOORS. They bind: operator approval and ``cluster activate`` +refuse below a floor, and at runtime volume creation, planned node shutdown +and node removal refuse when the cluster AFTER the operation would be below +one. The status monitor reports DEGRADED while the online resources are below +a floor. An outage may take a running cluster below the floors; nothing is +suspended for that reason alone -- the data plane keeps serving and +migrating, and rebalancing restores the protection. + +1. data devices per node >= 1 +2. data devices (cluster-wide, online) >= n+k when the cluster declares + ``raid_devices`` (every data device is protected below simplyblock), else + >= n+2k +3. failure domains (when failure domains are used at all) >= n+k +4. nodes >= n+2k +5. journal devices are excluded from every device count + +Topology overrides: a single-node cluster needs 1 node and n+k data devices; +a two-node (dual / arbitration) cluster needs 2 nodes. + +Multi-attach array LUN clusters (``cluster.multi_attach``, always 1+0 on +``raid_devices``) replace the node floor by FTT+1 (at least 3 for FTT 2, +``cluster.max_fault_tolerance`` = FTT): an owner's LUN is reopened on one of +the surviving nodes. One data LUN per node is the recommended layout; only +>= 1 is enforced. The topology overrides do not apply to them. + +Journal devices and the device model (floor 5) +---------------------------------------------- +A node's journal lives in ``StorageNode.jm_device`` (a ``JMDevice``), never in +``StorageNode.nvme_devices``: ``_prepare_cluster_devices_jm_on_dev`` moves the +device flagged ``NVMeDevice.STATUS_JM`` out of ``nvme_devices`` into +``jm_device`` and ``_prepare_cluster_devices_on_restart`` drops a STATUS_JM +entry from the list again. The three journal shapes therefore count like this: + +* partition journal nodes (``num_partitions_per_dev`` > 0, nvme and lblk + journal-split alike): every device in ``nvme_devices`` is a data device or + data partition; the journal partition(s) are only referenced from + ``jm_device`` and do not count; +* ``--enable-journal-device`` / ``--lblk-journal-device`` nodes: the whole + device that became the journal is ``jm_device`` and does not count; +* lblk journal-split nodes: the data partition is an ``nvme_devices`` entry + (``is_partition`` True) and counts; the journal partition is ``jm_device``. + +``count_data_devices`` nevertheless skips a device whose status is STATUS_JM +explicitly, for the window between flagging and preparation. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode + +#: Device statuses that count as an online data device. The same set +#: ``_cluster_activate`` counts (READONLY and CANNOT_ALLOCATE devices still +#: hold their chunks and serve reads). +ONLINE_DEVICE_STATUSES = ( + NVMeDevice.STATUS_ONLINE, + NVMeDevice.STATUS_READONLY, + NVMeDevice.STATUS_CANNOT_ALLOCATE, +) + + +@dataclass(frozen=True) +class Floors: + """The binding minimums of one cluster (or one volume scheme on it).""" + + nodes: int + devices_per_node: int + data_devices: int + #: None when failure domains are not used on the cluster. + failure_domains: int | None + #: Human-readable formula behind each floor, for the violation text. + nodes_rule: str = "n+2k" + data_devices_rule: str = "n+2k" + failure_domains_rule: str = "n+k" + devices_per_node_rule: str = "data devices per node" + + +def _flag(obj, name: str) -> bool: + """A boolean model flag read defensively: a MagicMock stand-in or a record + that predates the field must read as False, never as truthy.""" + return getattr(obj, name, False) is True + + +def _int(value) -> int: + """An integer model field read defensively (see ``_flag``): a stand-in + that is not an int reads as 0, which no floor can be below.""" + return value if isinstance(value, int) and not isinstance(value, bool) else 0 + + +def scheme(cluster, ndcs: int | None = None, npcs: int | None = None) -> tuple[int, int]: + """``(n, k)``: the volume override when one is given, else the cluster's.""" + if ndcs or npcs: + return _int(ndcs), _int(npcs) + return _int(getattr(cluster, "distr_ndcs", 0)), _int(getattr(cluster, "distr_npcs", 0)) + + +def is_single_node_cluster(cluster, member_nodes: int | None) -> bool: + """Flagged single-node, activated as single-node, or one member.""" + return (_flag(cluster, "is_single_node") + or _flag(cluster, "activated_single_node") + or member_nodes == 1) + + +def is_two_node_cluster(cluster, member_nodes: int | None) -> bool: + """A dual-node cluster is one whose whole membership is two nodes + (``storage_node_ops.apply_jc_dual_node`` keys the JC dual-node flag on + exactly that); ``cluster.two_node_arbitration`` is the arbitrated variant + and only ever set on such a cluster.""" + return _flag(cluster, "two_node_arbitration") or member_nodes == 2 + + +def cluster_floors(cluster, uses_failure_domains: bool, member_nodes: int | None = None, + ndcs: int | None = None, npcs: int | None = None) -> Floors: + """The floors of ``cluster`` for the scheme n+k (the cluster's, or the + volume override ``ndcs``/``npcs``). + + ``member_nodes`` is the number of primary storage nodes that belong to + the cluster (every status but REMOVED): it decides the single-node and + two-node topology overrides together with the cluster flags. + ``uses_failure_domains`` is ``cluster.enable_failure_domain``. + """ + n, k = scheme(cluster, ndcs, npcs) + raid = _flag(cluster, "raid_devices") + + if raid: + data_devices, data_rule = n + k, "n+k, raid_devices" + else: + data_devices, data_rule = n + 2 * k, "n+2k" + nodes, nodes_rule = n + 2 * k, "n+2k" + + if _flag(cluster, "multi_attach"): + ftt = _int(getattr(cluster, "max_fault_tolerance", 1)) + ftt = ftt if ftt in (1, 2) else 1 + return Floors( + nodes=max(ftt + 1, 3 if ftt == 2 else 0), + devices_per_node=1, + data_devices=n + k, + failure_domains=(n + k) if uses_failure_domains else None, + nodes_rule=f"FTT+1, multi-attach FTT {ftt}", + data_devices_rule="n+k, multi-attach", + devices_per_node_rule=("data devices per node; multi-attach: exactly one " + "data LUN per node is recommended"), + ) + + if is_single_node_cluster(cluster, member_nodes): + nodes, nodes_rule = 1, "single-node cluster" + # No redundancy to rebuild onto a spare: the stripe alone. + data_devices, data_rule = n + k, "n+k, single-node cluster" + elif is_two_node_cluster(cluster, member_nodes): + nodes, nodes_rule = 2, "two-node cluster" + + return Floors( + nodes=nodes, + devices_per_node=1, + data_devices=data_devices, + failure_domains=(n + k) if uses_failure_domains else None, + nodes_rule=nodes_rule, + data_devices_rule=data_rule, + ) + + +def is_data_device(dev) -> bool: + """An online data device: never the journal (floor 5, see the module + docstring), only the statuses that hold and serve chunks.""" + status = getattr(dev, "status", None) + if status == NVMeDevice.STATUS_JM: + return False + return status in ONLINE_DEVICE_STATUSES + + +def count_data_devices(nodes) -> tuple[int, dict[str, int]]: + """``(total, per_node)`` of online, non-journal data devices on ``nodes``. + + Every node given is counted (the caller decides which nodes are "online" + or "left after the operation"); a secondary-only node has no devices of + its own and is skipped. ``per_node`` has an entry for EVERY counted node, + zero included, so floor 1 (>= 1 data device per node) can be checked from + it. + """ + total = 0 + per_node: dict[str, int] = {} + for node in nodes: + if _flag(node, "is_secondary_node"): + continue + count = sum(1 for dev in (getattr(node, "nvme_devices", None) or []) + if is_data_device(dev)) + per_node[node.get_id()] = count + total += count + return total, per_node + + +def online_failure_domains(nodes) -> int: + """Distinct failure-domain ids (>= 0; -1 means untagged) among ``nodes``.""" + return len({node.failure_domain for node in nodes + if isinstance(getattr(node, "failure_domain", -1), int) + and node.failure_domain >= 0}) + + +def floor_violations(floors: Floors, online_nodes: int, data_devices: int, + per_node: dict[str, int], online_fds: int | None) -> list[str]: + """Every floor the given online counts violate, as human-readable text + naming the floor, e.g. ``"nodes online 3 < floor 4 (n+2k)"``. Empty when + every floor holds. ``online_fds`` None means failure domains are not + used (or not known) and floor 3 is not judged.""" + violations: list[str] = [] + if online_nodes < floors.nodes: + violations.append( + f"nodes online {online_nodes} < floor {floors.nodes} ({floors.nodes_rule})") + for node_id, count in sorted(per_node.items()): + if count < floors.devices_per_node: + violations.append( + f"node {node_id} has {count} online data devices < floor " + f"{floors.devices_per_node} ({floors.devices_per_node_rule})") + if data_devices < floors.data_devices: + violations.append( + f"data devices online {data_devices} < floor {floors.data_devices} " + f"({floors.data_devices_rule})") + if floors.failure_domains is not None and online_fds is not None \ + and online_fds < floors.failure_domains: + violations.append( + f"failure domains online {online_fds} < floor {floors.failure_domains} " + f"({floors.failure_domains_rule})") + return violations + + +def primary_members(snodes) -> list: + """The cluster's primary storage nodes that are members: every status but + REMOVED, secondary-only nodes left out.""" + return [n for n in snodes + if n.status != StorageNode.STATUS_REMOVED and not _flag(n, "is_secondary_node")] + + +def online_members(snodes) -> list: + return [n for n in primary_members(snodes) if n.status == StorageNode.STATUS_ONLINE] + + +def evaluate(cluster, snodes, exclude_node_id: str | None = None, + ndcs: int | None = None, npcs: int | None = None, + online_nodes=None) -> list[str]: + """The floor violations of ``cluster`` judged on the ONLINE resources of + ``snodes`` (every storage node of the cluster). + + ``exclude_node_id`` judges the cluster as it would be after that node + left (a planned shutdown or removal): the node is left out of the online + counts, but it stays a member for the topology overrides -- shutting one + node of a three-node cluster down does not make it a two-node cluster. + ``online_nodes`` replaces the ONLINE selection when the caller already + narrowed it (the monitor leaves nodes it does not count out). + """ + members = primary_members(snodes) + if online_nodes is None: + online_nodes = online_members(snodes) + online = [n for n in online_nodes + if n.get_id() != exclude_node_id and not _flag(n, "is_secondary_node")] + uses_fd = _flag(cluster, "enable_failure_domain") + floors = cluster_floors(cluster, uses_fd, member_nodes=len(members), ndcs=ndcs, npcs=npcs) + total, per_node = count_data_devices(online) + fds = online_failure_domains(online) if uses_fd else None + return floor_violations(floors, len(online), total, per_node, fds) + + +def migration_affinity_floor(cluster) -> str: + """The data plane's placement anti-affinity floor for a migration: + ``"relaxed"`` on a cluster whose data devices are protected below + simplyblock (``raid_devices``: several chunks of a stripe may share a + device when no other device is writable), ``"device"`` otherwise.""" + return "relaxed" if _flag(cluster, "raid_devices") else "device" diff --git a/simplyblock_core/kms/__init__.py b/simplyblock_core/kms/__init__.py index d980e2ca65..f366d00819 100644 --- a/simplyblock_core/kms/__init__.py +++ b/simplyblock_core/kms/__init__.py @@ -17,9 +17,12 @@ __all__ = [ "KMS", + "HCPClient", + "KMIPClient", "KMIPKMSSettings", "KMSException", "KMSSettings", + "LocalKMS", "VaultKMSSettings", "backup_dek_path", "backup_kek_name", @@ -41,6 +44,12 @@ def backup_dek_path(cluster_id: str, backup_id: str) -> str: return f"cluster/{cluster_id}/backup/{backup_id}" +def array_secret_path(cluster_id: str, array_id: str) -> str: + """Where the credentials of a cluster's storage array live in the KMS + (multi-attach array LUN mode, simplyblock_core.array).""" + return f"cluster/{cluster_id}/array/{array_id}" + + def pool_kek_name(pool_id: str) -> str: return f"pool-{pool_id}" @@ -49,7 +58,12 @@ def backup_kek_name(backup_id: str) -> str: return f"backup-{backup_id}" -def create_kms_connection(cluster: "Cluster") -> KMS: +def create_kms_connection(cluster: "Cluster", *, cert_role: str | None = None) -> KMS: + """Open the cluster's KMS backend. + + ``cert_role`` overrides the configured Vault cert-auth role; the other + backends ignore it. + """ kms_settings = cluster.get_kms_settings() if kms_settings is None: return LocalKMS(cluster) @@ -75,7 +89,7 @@ def create_kms_connection(cluster: "Cluster") -> KMS: settings.tls_key, transit_mount=kms_settings.transit_mount, kv_mount=kms_settings.kv_mount, - cert_role=kms_settings.cert_role, + cert_role=cert_role or kms_settings.cert_role, ) return KMIPClient( diff --git a/simplyblock_core/kms/_base.py b/simplyblock_core/kms/_base.py index a43cfcbaab..93dcbfa4e5 100644 --- a/simplyblock_core/kms/_base.py +++ b/simplyblock_core/kms/_base.py @@ -1,11 +1,22 @@ from __future__ import annotations +import hashlib +import hmac +import json from abc import abstractmethod +from collections.abc import Mapping from contextlib import AbstractContextManager from types import TracebackType from pydantic import SecretStr +from ._exceptions import KMSException + +#: Version of the envelope ``wrap_secret`` produces. Bump on any change to the +#: payload layout; ``unwrap_secret`` refuses versions it does not know. +_SECRET_ENVELOPE_VERSION = 1 +_DIGEST_LEN = hashlib.sha256().digest_size + class KMS(AbstractContextManager): def __exit__( # Has to be defined to make the type-checker happy @@ -48,3 +59,126 @@ def create_key_encryption_key(self, name: str) -> None: ... @abstractmethod def delete_key_encryption_key(self, name: str) -> None: ... + + def ensure_key_encryption_key(self, name: str) -> None: + """Create the KEK ``name`` unless it already exists. + + Must never replace an existing key: everything wrapped under it would + become unreadable. Backends whose ``create_key_encryption_key`` is not + already idempotent override this. + """ + self.create_key_encryption_key(name) + + # -- Opaque secrets (credentials for external systems) ------------------ + # + # Unlike data encryption keys these are not random key material but small + # sets of named string fields (username/password, API token, PEM client + # certificate, ...). Only the wrapped form is ever persisted. + + def _wrap_bytes(self, kek_name: str, plaintext: bytes) -> str: + """Encrypt ``plaintext`` under the KEK, returning a storable string.""" + raise NotImplementedError(f"{type(self).__name__} cannot wrap secrets") + + def _unwrap_bytes(self, kek_name: str, wrapped: str) -> bytes: + """Inverse of :meth:`_wrap_bytes`.""" + raise NotImplementedError(f"{type(self).__name__} cannot unwrap secrets") + + def wrap_secret( + self, kek_name: str, fields: Mapping[str, SecretStr], context: str, + ) -> str: + """Wrap ``fields`` under ``kek_name``, bound to ``context``. + + ``context`` names what the secret is *for* (e.g. the external system's + identity and endpoint). :meth:`unwrap_secret` refuses to return the + secret for any other context, so a record whose endpoint was rewritten + in the database cannot be used to send the credential elsewhere. + + A SHA-256 digest of the payload is wrapped together with it. Vault + Transit (AES-GCM) already authenticates its ciphertext; the KMIP + backend wraps with AES-CBC (see ``_kmip.py``), which does not, and the + embedded digest turns blind tampering with that ciphertext into a + refusal instead of a silently altered secret or context. + """ + if not fields: + raise KMSException("Refusing to wrap an empty secret") + payload = json.dumps( + { + "v": _SECRET_ENVELOPE_VERSION, + "ctx": context, + "fields": {name: value.get_secret_value() for name, value in fields.items()}, + }, + sort_keys=True, separators=(",", ":"), + ).encode() + return self._wrap_bytes(kek_name, hashlib.sha256(payload).digest() + payload) + + def unwrap_secret( + self, kek_name: str, wrapped: str, context: str, + ) -> dict[str, SecretStr]: + """Inverse of :meth:`wrap_secret`; ``context`` must match exactly.""" + blob = self._unwrap_bytes(kek_name, wrapped) + digest, payload = blob[:_DIGEST_LEN], blob[_DIGEST_LEN:] + if len(digest) != _DIGEST_LEN or not hmac.compare_digest( + digest, hashlib.sha256(payload).digest(), + ): + raise KMSException("Wrapped secret failed its integrity check") + try: + data = json.loads(payload) + except ValueError as e: + raise KMSException("Wrapped secret is not valid") from e + if not isinstance(data, dict) or data.get("v") != _SECRET_ENVELOPE_VERSION: + raise KMSException("Wrapped secret has an unsupported format") + if not hmac.compare_digest(str(data.get("ctx", "")).encode(), context.encode()): + raise KMSException("Wrapped secret is bound to a different context") + fields = data.get("fields") + if not isinstance(fields, dict) or not fields or not all( + isinstance(k, str) and isinstance(v, str) for k, v in fields.items() + ): + raise KMSException("Wrapped secret is not valid") + return {name: SecretStr(value) for name, value in fields.items()} + + # Generic secrets: a flat ``{name: value}`` dict stored at ``path`` (array + # credentials, see simplyblock_core.array). Values go in as ``SecretStr`` + # (plain strings are wrapped) and come back as ``SecretStr``; only the + # persistence boundary of each implementation sees the plaintext. + + @abstractmethod + def put_secret(self, path: str, data: dict) -> None: + """Store ``data`` at ``path``, replacing whatever was there.""" + + @abstractmethod + def get_secret(self, path: str) -> dict: + """The secret stored at ``path``, every value a ``SecretStr``. + + Raises: + KMSException: nothing is stored at ``path``, or the read failed. + """ + + @abstractmethod + def delete_secret(self, path: str) -> None: + """Delete the secret at ``path``; deleting a missing one succeeds.""" + + +def secret_plaintext(data: dict) -> dict: + """``data`` with every value unwrapped to a plain string, for the + persistence boundary of a KMS only. Keys must be strings; values are + ``SecretStr`` or ``str`` (anything else is refused, so a nested structure + cannot carry an unmasked secret past the wrapper).""" + if not isinstance(data, dict): + raise TypeError("a secret is a dict of strings") + out = {} + for k, v in data.items(): + if not isinstance(k, str): + raise TypeError("secret keys must be strings") + if isinstance(v, SecretStr): + out[k] = v.get_secret_value() + elif isinstance(v, str): + out[k] = v + else: + raise TypeError(f"secret value of {k!r} must be a string") + return out + + +def secret_wrapped(data: dict) -> dict: + """``data`` read back from a KMS, every value wrapped in ``SecretStr``.""" + return {str(k): v if isinstance(v, SecretStr) else SecretStr(str(v)) + for k, v in (data or {}).items()} diff --git a/simplyblock_core/kms/_fdb.py b/simplyblock_core/kms/_fdb.py index b45e37c380..f2b94c07e0 100644 --- a/simplyblock_core/kms/_fdb.py +++ b/simplyblock_core/kms/_fdb.py @@ -1,3 +1,5 @@ +import base64 +import binascii import json from typing import TYPE_CHECKING @@ -5,7 +7,7 @@ from simplyblock_core.utils import generate_hex_string -from ._base import KMS +from ._base import KMS, secret_plaintext, secret_wrapped from ._exceptions import KMSException if TYPE_CHECKING: @@ -29,6 +31,12 @@ def __init__(self, cluster: "Cluster"): def _key(path: str) -> bytes: return f"keys/{path}".encode() + @staticmethod + def _secret_key(path: str) -> bytes: + # A prefix of its own: a generic secret never shares a key with a DEK + # pair stored at the same path. + return f"secrets/{path}".encode() + def create_data_encryption_keys(self, path: str, kek_name: str) -> None: self.import_data_encryption_keys( path, kek_name, @@ -60,3 +68,33 @@ def create_key_encryption_key(self, name: str) -> None: def delete_key_encryption_key(self, name: str) -> None: pass + + def put_secret(self, path: str, data: dict) -> None: + # The persistence boundary, like import_data_encryption_keys. + self._kv_store.set(self._secret_key(path), json.dumps(secret_plaintext(data)).encode()) + + def get_secret(self, path: str) -> dict: + raw = self._kv_store.get(self._secret_key(path)) + if not raw: + raise KMSException(f"No secret found at {path}") + return secret_wrapped(json.loads(raw)) + + def delete_secret(self, path: str) -> None: + self._kv_store.clear(self._secret_key(path)) + + # There is no KEK here: the "wrapped" form is only encoded, not encrypted, + # so a secret wrapped by LocalKMS is effectively plaintext in FoundationDB. + # Credential storage gates this behind an explicit per-cluster opt-in + # (see simplyblock_core.credentials). + _LOCAL_PREFIX = "local:" + + def _wrap_bytes(self, kek_name: str, plaintext: bytes) -> str: + return self._LOCAL_PREFIX + base64.b64encode(plaintext).decode() + + def _unwrap_bytes(self, kek_name: str, wrapped: str) -> bytes: + if not wrapped.startswith(self._LOCAL_PREFIX): + raise KMSException("Wrapped secret was not produced by the local KMS") + try: + return base64.b64decode(wrapped[len(self._LOCAL_PREFIX):], validate=True) + except binascii.Error as e: + raise KMSException("Wrapped secret is not valid") from e diff --git a/simplyblock_core/kms/_hcp.py b/simplyblock_core/kms/_hcp.py index a68d56b241..593b22b918 100644 --- a/simplyblock_core/kms/_hcp.py +++ b/simplyblock_core/kms/_hcp.py @@ -6,7 +6,7 @@ import hvac.exceptions from pydantic import SecretStr -from ._base import KMS +from ._base import KMS, secret_plaintext, secret_wrapped from ._exceptions import KMSException logger = logging.getLogger(__name__) @@ -137,3 +137,88 @@ def delete_key_encryption_key(self, name: str) -> None: self.client.secrets.transit.delete_key(name=name, mount_point=self.transit_mount) except hvac.exceptions.VaultError as e: raise KMSException("Request failed") from e + + # Generic secrets live in the kv v2 mount under the path itself (DEKs use + # the same mount, under cluster//lvol|backup/...; array credentials + # under cluster//array/, so the two never collide). + + def put_secret(self, path: str, data: dict) -> None: + try: + self.client.secrets.kv.v2.create_or_update_secret( + path=path, secret=secret_plaintext(data), mount_point=self.kv_mount, + ) + except hvac.exceptions.VaultError as e: + raise KMSException("Request failed") from e + + def get_secret(self, path: str) -> dict: + try: + data = self.client.secrets.kv.v2.read_secret_version( + path=path, mount_point=self.kv_mount, raise_on_deleted_version=True, + )['data']['data'] + except hvac.exceptions.InvalidPath as e: + raise KMSException(f"No secret found at {path}") from e + except hvac.exceptions.VaultError as e: + raise KMSException("Request failed") from e + return secret_wrapped(data) + + def delete_secret(self, path: str) -> None: + try: + self.client.secrets.kv.v2.delete_metadata_and_all_versions( + path=path, mount_point=self.kv_mount, + ) + except hvac.exceptions.InvalidPath: + return + except hvac.exceptions.VaultError as e: + raise KMSException("Request failed") from e + + def ensure_key_encryption_key(self, name: str) -> None: + try: + self.client.secrets.transit.read_key(name=name, mount_point=self.transit_mount) + return + except hvac.exceptions.InvalidPath: + pass + except hvac.exceptions.VaultError as e: + raise KMSException("Request failed") from e + self.create_key_encryption_key(name) + + def _wrap_bytes(self, kek_name: str, plaintext: bytes) -> str: + try: + return self.client.secrets.transit.encrypt_data( + name=kek_name, plaintext=base64.b64encode(plaintext).decode(), + mount_point=self.transit_mount, + )['data']['ciphertext'] + except hvac.exceptions.VaultError as e: + raise KMSException("Request failed") from e + + def _unwrap_bytes(self, kek_name: str, wrapped: str) -> bytes: + try: + plaintext_b64 = self.client.secrets.transit.decrypt_data( + name=kek_name, ciphertext=wrapped, mount_point=self.transit_mount, + )['data']['plaintext'] + except hvac.exceptions.VaultError as e: + raise KMSException("Request failed") from e + return base64.b64decode(plaintext_b64) + + def read_kv_secret( + self, mount: str, path: str, version: int | None = None, + ) -> dict[str, SecretStr]: + """Read a customer-managed KV v2 secret as named string fields. + + Errors name the path and field names, never values. + """ + try: + data = self.client.secrets.kv.v2.read_secret_version( + path=path, mount_point=mount, version=version, + raise_on_deleted_version=True, + )['data']['data'] + except hvac.exceptions.InvalidPath as e: + raise KMSException(f"No secret at {mount}/{path}") from e + except hvac.exceptions.VaultError as e: + raise KMSException("Request failed") from e + if not isinstance(data, dict) or not data: + raise KMSException(f"Secret at {mount}/{path} is empty") + bad = sorted(name for name, value in data.items() if not isinstance(value, str)) + if bad: + raise KMSException( + f"Secret at {mount}/{path} has non-string fields: {', '.join(bad)}") + return {name: SecretStr(value) for name, value in data.items()} diff --git a/simplyblock_core/kms/_kmip.py b/simplyblock_core/kms/_kmip.py index 21a50dead2..ffa0bb3d4a 100644 --- a/simplyblock_core/kms/_kmip.py +++ b/simplyblock_core/kms/_kmip.py @@ -3,6 +3,7 @@ from kmip.core import enums from kmip.pie import exceptions as kmip_exceptions +from kmip.pie import objects as kmip_objects from kmip.pie.client import ProxyKmipClient from pydantic import SecretStr @@ -133,9 +134,12 @@ def delete_key_encryption_key(self, name: str) -> None: kv_store.clear(self._kek_key(name)) def _encrypt(self, kek_uid: str, plaintext_hex: SecretStr) -> dict: + return self._encrypt_bytes(kek_uid, bytes.fromhex(plaintext_hex.get_secret_value())) + + def _encrypt_bytes(self, kek_uid: str, plaintext: bytes) -> dict: try: ciphertext, iv_counter_nonce = self.client.encrypt( - bytes.fromhex(plaintext_hex.get_secret_value()), + plaintext, uid=kek_uid, cryptographic_parameters=dict(_CRYPTOGRAPHIC_PARAMETERS), ) @@ -147,6 +151,9 @@ def _encrypt(self, kek_uid: str, plaintext_hex: SecretStr) -> dict: } def _decrypt(self, kek_uid: str, wrapped: dict) -> SecretStr: + return SecretStr(self._decrypt_bytes(kek_uid, wrapped).hex()) + + def _decrypt_bytes(self, kek_uid: str, wrapped: dict) -> bytes: try: plaintext = self.client.decrypt( bytes.fromhex(wrapped["ciphertext"]), @@ -156,7 +163,7 @@ def _decrypt(self, kek_uid: str, wrapped: dict) -> SecretStr: ) except _KMIP_ERRORS as e: raise KMSException("Request failed") from e - return SecretStr(plaintext.hex()) + return plaintext def create_data_encryption_keys(self, path: str, kek_name: str) -> None: self.import_data_encryption_keys( @@ -181,3 +188,57 @@ def get_data_encryption_keys(self, path: str, kek_name: str) -> tuple[SecretStr, def delete_data_encryption_keys(self, path: str) -> None: self._kv_store().clear(self._dek_key(path)) + + # Generic secrets (the array credentials of a multi-attach cluster, + # simplyblock_core.controllers.array_controller). Not supported on KMIP + # yet: KMIP keeps keys, and storing secrets would mean wrapping them with + # an appliance KEK the way the DEKs above are, which has not been tested + # against a KMIP appliance. Refused explicitly rather than half-done. + _NO_SECRETS = ("The KMIP key management backend does not store array credentials yet; " + "use HashiCorp Vault / OpenBao or the local KMS for a multi-attach cluster " + "with array credentials, or register the array without credentials " + "(e.g. aws-ebs with an instance role)") + + def put_secret(self, path: str, data: dict) -> None: + raise KMSException(self._NO_SECRETS) + + def get_secret(self, path: str) -> dict: + raise KMSException(self._NO_SECRETS) + + def delete_secret(self, path: str) -> None: + raise KMSException(self._NO_SECRETS) + + def ensure_key_encryption_key(self, name: str) -> None: + # create_key_encryption_key always mints a new appliance key and + # repoints the name at it, which would orphan everything already + # wrapped under the old one. + if self._kv_store().get(self._kek_key(name)): + return + self.create_key_encryption_key(name) + + def _wrap_bytes(self, kek_name: str, plaintext: bytes) -> str: + return json.dumps(self._encrypt_bytes(self._kek_uid(kek_name), plaintext)) + + def _unwrap_bytes(self, kek_name: str, wrapped: str) -> bytes: + try: + parsed = json.loads(wrapped) + if not isinstance(parsed, dict) or "ciphertext" not in parsed: + raise ValueError("not a wrapped KMIP record") + return self._decrypt_bytes(self._kek_uid(kek_name), parsed) + except ValueError as e: # bad JSON, or bytes.fromhex on a corrupted record + raise KMSException("Wrapped secret is not valid") from e + + def get_secret_data(self, uid: str) -> bytes: + """Fetch a customer-managed KMIP ``SecretData`` object's raw value. + + Support for ``SecretData`` varies between appliances (some only manage + keys); a server that refuses it surfaces as ``KMSException``. + """ + try: + obj = self.client.get(uid) + except _KMIP_ERRORS as e: + raise KMSException(f"Could not fetch KMIP object {uid}") from e + if not isinstance(obj, kmip_objects.SecretData): + raise KMSException( + f"KMIP object {uid} is a {type(obj).__name__}, not SecretData") + return bytes(obj.value) diff --git a/simplyblock_core/kms/_settings.py b/simplyblock_core/kms/_settings.py index ca1c92d992..fa9ebf21d9 100644 --- a/simplyblock_core/kms/_settings.py +++ b/simplyblock_core/kms/_settings.py @@ -29,6 +29,11 @@ class VaultKMSSettings(BaseModel): transit_mount: str = "simplyblock/transit" kv_mount: str = "simplyblock/kv" cert_role: str = "simplyblock-webappapi" + #: Vault cert-auth role used for credentials of external systems (storage + #: arrays, ...) instead of ``cert_role``, so a Vault policy can grant those + #: paths separately from volume encryption keys. ``None`` reuses + #: ``cert_role``. + credentials_cert_role: str | None = None #: Same convention as BackupLocation._serialize_endpoint: the field owns #: its own JSON-safety, so plain model_dump() is enough here too. diff --git a/simplyblock_core/models/base_model.py b/simplyblock_core/models/base_model.py index 13f3243a7f..8d461350fb 100644 --- a/simplyblock_core/models/base_model.py +++ b/simplyblock_core/models/base_model.py @@ -503,6 +503,13 @@ def _apply_index_diff(tr, model_cls, index_list, old_keys, obj, *, @staticmethod def _write_tx(tr, key, value, model_cls, obj, index_list, rollup_key, version_key): + merge = getattr(model_cls, "merge_on_full_write", None) + if merge is not None and obj.needs_full_write_merge(): + # Fields only atomic_update may change keep their stored value: a + # full-object write of a stale copy must not revert them. + raw = tr.get(key).wait() + if raw is not None and raw.present(): + value = merge(json.loads(bytes(raw)), value) if index_list: BaseModel._apply_index_diff( tr, model_cls, index_list, diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index 1cb0a843a2..3aea4cd80e 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -216,6 +216,66 @@ def is_topology_owned(self) -> bool: node_label_suffix: str = "" disable_monitoring: bool = False strict_node_anti_affinity: bool = False + # Cluster-create-time, immutable declaration that every data device is + # protected below simplyblock (hardware RAID LUN, md or dm RAID). Lowers + # the data-device floor from n+2k to n+k (simplyblock_core.floors) and + # lets the data plane place several chunks of a stripe on one device when + # no other device is writable (allow_device_colocation on the distribs, + # affinity_floor "relaxed" on migrations). + raid_devices: bool = False + # Cluster-create-time, immutable: multi-attach array LUN mode. Every data + # device is an array LUN presented to several storage nodes; the cluster + # runs 1+0 and tolerates max_fault_tolerance node failures by reopening + # an owner's LUN on another node and adding it as an extra NVMe-oF path. + # Requires raid_devices. Gates every multi-attach path (device NQN, + # controller-ID ranges, hold timeouts, owner loopback). + multi_attach: bool = False + # Multi-attach clusters, opt-in: keep a warm standby alceml (standby=true, + # page map kept warm in hugepages) of every LUN on each attached + # non-serving node, so a takeover activates it on the first I/O instead of + # loading the whole page map cold (lun_takeover.ensure_warm_standbys). + # Costs the page-map memory of every such LUN on each standby node. + multi_attach_warm_standby: bool = False + # Multi-attach clusters: the storage array the LUNs live on. A LUN is + # taken over by another node only after its serving node is fenced at the + # array, through the plugin named here (entry point in the group + # simplyblock.array_plugins, see simplyblock_core.array). Empty = no + # plugin: a takeover is refused (never reopen a LUN unfenced). The array + # is registered through controllers/array_controller (POST + # /clusters/{id}/arrays). Its credentials are never kept here, only a + # reference to them (array_credential_ref). + array_plugin: str = "" + array_endpoint: str = "" + array_options: dict = default_factory(dict) + # Id of the registered array ("" = none, or configured before arrays were + # registered) and whether credentials were stored for it in the KMS. + array_id: str = "" + array_has_credentials: bool = False + # Where the array's credentials are (simplyblock_core.credentials): a + # "managed" reference (wrapped by the cluster's credentials key in its + # KMS, bound to array id + endpoint), or an external "vault-kv" / + # "kmip-secret" secret. {} with array_has_credentials = an array + # registered before references existed: its secret is in the KMS at + # cluster//array/. + array_credential_ref: dict = default_factory(dict) + # Size in bytes of the LUN created for each node added to the cluster + # while an array is registered; 0 = node adds create no LUN (the LUNs are + # presented by the operator, as in phase 2/3). + array_lun_size: int = 0 + # Where each node's journal is kept when the array plugin creates the + # nodes' LUNs (array_lun_size > 0): + # - "local": a disk or partition of the node's host, named at sn configure + # (--lblk-journal-only); never shared. + # - "lun": a LUN of its own the plugin creates at node add, next to the + # data LUN, presented to that node only (never multi-attached). It is + # created with array_options updated by array_journal_options (e.g. + # {"pool": "fast-pool"}: a pool of higher performance or priority) and + # array_journal_lun_size bytes (0 = 3% of array_lun_size, at least + # 2 GiB). The node's host then needs no local device + # (sn configure --lblk-array-only). + array_journal: str = "local" + array_journal_lun_size: int = 0 + array_journal_options: dict = default_factory(dict) tls: bool = False tls_config: dict = default_factory(dict) is_re_balancing: bool = False @@ -430,17 +490,18 @@ def is_topology_owned(self) -> bool: # migration for every create/update. Empty dict = no external KMS (LocalKMS). kms_settings: dict = default_factory(dict) hashicorp_vault_settings: HashicorpVaultSettings | None = None + # Explicit opt-in to keep credentials for external systems (storage + # arrays, ...) with no external KMS configured. LocalKMS only encodes, so + # such credentials would sit effectively in plaintext in FoundationDB; + # without this flag simplyblock_core.credentials refuses to store or read + # them unless a Vault or KMIP backend is configured. + allow_local_kms_credentials: bool = False # Single-node-expansion resumability cursor. Empty dict means no expansion # is in flight. Populated/advanced/cleared via the helpers in # ``simplyblock_core.controllers.cluster_expansion.planner``. See the feature plan # ``single_node_expansion_plan.md`` for the schema. expand_state: dict = default_factory(dict) - # Set while new device weights (placement sizes after an uneven growth of - # lblk devices) are written but not yet pushed to every node and handed to - # a rebalance. A re-run of `cluster grow-devices` finishes the job. See - # device_controller.rebalance_after_growth. - reweight_pending: bool = False # State owned by release-specific upgrade plug-ins (see # simplyblock_core/release_upgrades/). Keys are plugin STATE_KEYs; a key # is set by the plugin's pre_update step (first thing `cluster update` diff --git a/simplyblock_core/models/nvme_device.py b/simplyblock_core/models/nvme_device.py index 76690d2031..ec2f92ec15 100644 --- a/simplyblock_core/models/nvme_device.py +++ b/simplyblock_core/models/nvme_device.py @@ -77,9 +77,17 @@ class NVMeDevice(BaseModel): # distrib placement weight of the device is derived from it, and distrib # requires a weight never to change once set (a change re-places 50% or # more of the data). ``size`` follows an online growth of an lblk device - # (device_controller.resize_device); this field keeps the weight frozen. - # 0 means the device was never resized, so ``size`` is still the weight. + # (device_controller.grow_cluster_devices); this field keeps the weight + # frozen. 0 means the device was never grown, so ``size`` is still the + # weight. placement_size: int = 0 + # Size in bytes of the device's block device (the LUN) when the cluster + # last rescanned it (device_controller.grow_cluster_devices); 0 = never + # rescanned. alceml always uses the whole block device, but all devices + # grow by the same factor only, so ``size`` may stay below this value; the + # rest is unused and left out of the capacity. A value below ``size`` means + # the block device shrank, which raises an alert. + backend_size: int = 0 # NVMe per-block metadata size in bytes, as reported by the bound SPDK bdev. # >=8 means alceml can run in cv_md_method (no read/write amplification). # 0 means alceml must use cv_fallback_method (extra md page per 2 MiB extent). @@ -105,6 +113,22 @@ class NVMeDevice(BaseModel): # Stable /dev/disk/by-id/... symlink when the device has one; preferred # as the AIO bdev filename so udev renames cannot bite mid-flight. by_id_path: str = "" + # Block size in bytes the AIO base bdev of an lblk device was created + # with, and must be re-created with on every restart: alceml's on-disk + # layout depends on the base bdev block size, so data written at one + # size cannot be read at another. 0 means the bdev was created without a + # block size (SPDK used the device's logical block size): every device + # created before this field existed, which keeps being created that way. + # Also kept in the node's ``lblk_devices`` entries (see + # utils.recorded_aio_block_size). + aio_block_size: int = 0 + # I/O engine of the base bdev of an lblk device: "aio" (bdev_aio) or + # "uring" (bdev_uring, for device-mapper nodes such as multipath maps, + # which refuse the RWF_NOWAIT submits of bdev_aio). Only the + # create/delete/rescan RPCs differ; the bdev name and bdev_type "aio" + # are the same. Does not change the on-disk format. Also kept in the + # node's ``lblk_devices`` entries (see utils.recorded_lblk_io_engine). + lblk_io_engine: str = "aio" # The latest disk-health assessment an external assessor (the Kubernetes # operator) recorded through PUT .../devices/{id}/health-assessment, # stored verbatim: {"reporter", "reported_at", "valid_seconds", @@ -112,12 +136,61 @@ class NVMeDevice(BaseModel): # does not compute it; alerts_controller turns it into device_slow / # device_unhealthy while it is fresh. Empty until first reported. health_assessment: dict = default_factory(dict) + # Multi-attach array LUN mode (cluster.multi_attach). All "off" by + # default, so a device of a regular cluster is unchanged. + #: The device is an array LUN presented to several storage nodes; its + #: subsystem NQN is host-independent (``:lun:``) and + #: every distrib, the owner's included, reaches it as + #: ``remote_alceml_n1`` over NVMe-oF. + multi_attach: bool = False + #: World-wide name (or serial) of the LUN, the same on every node. + lun_wwn: str = "" + #: Node whose alceml currently serves the LUN; "" means the owner + #: (``node_id``). + serving_node_id: str = "" + #: node_id -> {"path": by-id path of the LUN on that node, "bdev": the + #: base bdev name there}. The owner's entry, plus the LVS secondary and + #: tertiary once an array-managed LUN is attached to them (bdev ""). + node_paths: dict = default_factory(dict) + #: Nodes a LUN takeover fenced from this LUN at the array. Phase 3 never + #: unfences on its own: a fenced node's host may still hold queued + #: writes. Such a node is no takeover candidate for this LUN. + fenced_node_ids: list = default_factory(list) + #: The LUN's alceml instances run with persistent reservation keys + #: (multi_attach.reservation_key): the serving one holds the reservation + #: and a takeover fences by preempting it. Decided once, when the device + #: is first marked a LUN (multi_attach.mark_lun_devices): a LUN formatted + #: with a key cannot be opened without one. + lun_reservations: bool = False + #: The array's id of the LUN when the cluster's array plugin created it + #: (controllers/array_controller); "" for a LUN the operator presented. + #: Only such LUNs are attached, detached, grown and deleted by simplyblock. + array_lun_id: str = "" + #: Nodes the array presents the LUN to, as last applied by + #: array_controller.reconcile_lun_attachments (owner, LVS secondary and + #: tertiary, serving node). + array_attached_node_ids: list = default_factory(list) + + def serving_node(self) -> str: + """The node whose alceml serves this device now (the owner unless a + multi-attach takeover moved it).""" + return self.serving_node_id or self.node_id def weight_size(self) -> int: """Bytes the cluster-map weight of this device is derived from: the size it had when it was placed, unchanged by later growth.""" return self.placement_size if self.placement_size > 0 else self.size + def unused_size(self) -> int: + """Bytes of the block device beyond ``size``: grown on the storage + array, but not usable because the other data devices of the cluster + grew by a smaller factor.""" + return max(0, self.backend_size - self.size) if self.size > 0 else 0 + + def is_shrunk(self) -> bool: + """The block device was last seen smaller than the device.""" + return 0 < self.backend_size < self.size + def __change_dev_connection_to(self, connecting_from_node): # Targeted single-record write. The previous implementation scanned # the WHOLE node table and full-object-wrote the owning node record diff --git a/simplyblock_core/models/storage_node.py b/simplyblock_core/models/storage_node.py index 32d00b0634..f4d2b3bacf 100644 --- a/simplyblock_core/models/storage_node.py +++ b/simplyblock_core/models/storage_node.py @@ -1,4 +1,5 @@ import hashlib +import json import time from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, ClassVar @@ -42,6 +43,45 @@ class StorageNode(BaseNodeObject): _WATCHED = True + #: Multi-attach LUN ownership fields of a device. Only atomic_update + #: changes them (lun_takeover: takeover, fence, unfence; array_controller: + #: attachment reconcile); a full-object write keeps their stored values. + #: A restart's full writes from a copy loaded before an unfence restored + #: fenced_node_ids, and the reintegrated node stayed fenced for good + #: (2026-11-20 FTT2 bed, net cut + SPDK kill of the owner). + LUN_OWNERSHIP_FIELDS = ("serving_node_id", "fenced_node_ids", "array_attached_node_ids") + + def needs_full_write_merge(self) -> bool: + """Only nodes with multi-attach devices pay the stored-record read + (every other node's write is unchanged).""" + return any(getattr(d, "multi_attach", False) is True for d in self.nvme_devices or []) + + @staticmethod + def merge_on_full_write(stored: dict, value: bytes) -> bytes: + """``value`` (a full-object write) with the stored LUN ownership + fields of every multi-attach device present in both.""" + old = {d.get("uuid"): d for d in stored.get("nvme_devices") or [] + if isinstance(d, dict) and d.get("multi_attach") is True} + if not old: + return value + new = json.loads(value) + kept = [] + for d in new.get("nvme_devices") or []: + o = old.get(d.get("uuid")) if isinstance(d, dict) else None + if o is None: + continue + for f in StorageNode.LUN_OWNERSHIP_FIELDS: + if f in o and d.get(f) != o.get(f): + kept.append(f"{d.get('uuid')}.{f}: {d.get(f)!r} -> kept {o.get(f)!r}") + d[f] = o[f] + if not kept: + return value + from simplyblock_core import utils + utils.get_logger(__name__).warning( + "[NODE-WRITE] full-object write of %s from a stale copy: LUN ownership kept from the " + "stored record (%s)", new.get("uuid"), "; ".join(kept)) + return json.dumps(new).encode() + # A JM built on a whole device inherits that device's uuid # (storage_node_ops._create_jm_stack_on_device), so a bare device id does # not say which kind of device it names. The kind is the second segment of @@ -230,6 +270,18 @@ class StorageNode(BaseNodeObject): # placement and emitted verbatim into the distrib cluster map for the data # plane. failure_domain: int = -1 + # Multi-attach array LUN clusters only: this node's stable controller-ID + # block (multi_attach.cntlid_range). -1 = not assigned; claimed at + # add-node or lazily when a multi-attach device subsystem is created. + cntlid_block: int = -1 + # Multi-attach clusters with a registered array: the array's id of this + # node's host (ArrayPlugin.ensure_host; the EC2 instance id on EBS). "" + # = not known; the fence then names the host by its hostname. + array_host_id: str = "" + # Cluster.array_journal "lun": the array's id of this node's journal LUN + # ("" = its journal is local). Released with the node, deleted with the + # cluster, like its data LUN. + array_journal_lun_id: str = "" # Sync-replication site of the node (operator-supplied at add-node). # Mandatory on a cluster with sync_replication, empty everywhere else. site: str = "" diff --git a/simplyblock_core/multi_attach.py b/simplyblock_core/multi_attach.py new file mode 100644 index 0000000000..aa7e35ed37 --- /dev/null +++ b/simplyblock_core/multi_attach.py @@ -0,0 +1,443 @@ +"""Multi-attach array LUN mode (``Cluster.multi_attach``), phase 2 helpers. + +On an array whose LUNs are already protected and presented to several storage +nodes, the cluster runs 1+0. Each node owns one data LUN; exactly one alceml +per LUN is active. Everything a peer (and, through a loopback attach, the owner +itself) uses to reach the LUN is an NVMe-oF controller ``remote_alceml_`` +whose subsystem NQN does not depend on the exporting host, so a later takeover +can add the LUN's export on another node as one more path of the same +controller: + +* the device subsystem NQN is ``:lun:`` + (SPDK refuses a path to a different subnqn); +* each node exports with its own controller-ID range (SPDK multipath refuses + duplicate cntlids across the paths of one controller); +* peers attach with a long per-controller hold, so I/O waits for a new path + instead of failing. + +Every helper here is a no-op / returns the regular value for a device or +cluster without the flag, so regular clusters are unchanged. +""" + +from __future__ import annotations + +import logging +import uuid as uuid_mod + +from simplyblock_core import constants + +logger = logging.getLogger(__name__) + +#: The NQN infix that marks a host-independent multi-attach device subsystem. +LUN_NQN_INFIX = ":lun:" + + +def is_multi_attach(obj) -> bool: + """The multi-attach flag of a cluster or device, read defensively: a + record that predates the field or a test stand-in reads as False.""" + return getattr(obj, "multi_attach", False) is True + + +def device_nqn(cluster_nqn: str, device_id: str) -> str: + """The host-independent subsystem NQN of a multi-attach device: the base + of the cluster NQN (the part every node's ``snode.subsystem`` starts with) + plus ``:lun:``.""" + base = (cluster_nqn or constants.CLUSTER_NQN).split(":")[0] + return f"{base}{LUN_NQN_INFIX}{device_id}" + + +def device_subsystem_nqn(snode, device, cluster) -> str: + """The subsystem NQN a device is exported under: host-independent for a + multi-attach device, ``:dev:`` otherwise.""" + if is_multi_attach(device): + return device_nqn(getattr(cluster, "nqn", ""), device.get_id()) + return snode.subsystem + ":dev:" + device.get_id() + + +#: Namespace of the derived namespace UUIDs (``namespace_uuid``); fixed forever. +_LUN_NS_UUID_NAMESPACE = uuid_mod.UUID("6f1d3a52-8c4b-5e0a-9d27-3b8e4c1f7a90") + + +def namespace_uuid(device) -> str: + """The namespace UUID a device's PT bdev is exported with. + + Regular device: the device UUID, as before. Multi-attach device: a UUID + derived from the device UUID, the same on every node that exports the LUN + (SPDK multipath only joins paths whose namespaces are identical). It must + NOT be the device UUID itself: the alceml bdev is registered with the + device UUID, SPDK registers every bdev UUID as an alias, and the owner's + loopback controller (and a takeover node's controller to the LUN) would + then fail to register its namespace bdev with a duplicate alias. + """ + if not is_multi_attach(device): + return device.get_id() + return str(uuid_mod.uuid5(_LUN_NS_UUID_NAMESPACE, device.get_id())) + + +def namespace_nguid(device): + """The NGUID a multi-attach device's namespace is exported with: the + namespace UUID's 32 hex digits. None for a regular device (SPDK then + derives it from the bdev, as before). + + Without it SPDK fills the NGUID from the exported PT bdev's UUID, and the + host side registers its namespace bdev under that NGUID. The ptnonexcl PT + bdev's UUID is derived from its name (bdev_ptnonexcl.c), the same name on + every node, so on any node that both holds the PT bdev and consumes the + LUN (the owner's loopback, a takeover node) the consumer bdev's UUID + collides with the PT bdev's and SPDK refuses to register it (found on the + multi-attach bed: "Unable to add uuid: alias for bdev + remote_alceml_n1").""" + if not is_multi_attach(device): + return None + return uuid_mod.UUID(namespace_uuid(device)).hex.upper() + + +def cntlid_range(block: int) -> tuple[int, int]: + """``(min_cntlid, max_cntlid)`` of controller-ID block ``block``. + + Raises: + ValueError: ``block`` is negative or beyond the supported node count. + """ + if not isinstance(block, int) or block < 0 or block >= constants.MULTI_ATTACH_MAX_NODES: + raise ValueError( + f"controller-ID block {block!r} out of range [0, {constants.MULTI_ATTACH_MAX_NODES})") + size = constants.MULTI_ATTACH_CNTLID_BLOCK_SIZE + lo, hi = block * size + 1, block * size + size + if hi > constants.MULTI_ATTACH_MAX_CNTLID: + raise ValueError(f"controller-ID block {block} exceeds cntlid 0x{constants.MULTI_ATTACH_MAX_CNTLID:X}") + return lo, hi + + +def ensure_cntlid_block(snode, db_controller=None) -> int: + """The node's controller-ID block, claimed (smallest free index of the + cluster, persisted on the node record) when it has none yet. Sets + ``snode.cntlid_block`` in memory too, so a later full write of this + object keeps it.""" + block = getattr(snode, "cntlid_block", -1) + if isinstance(block, int) and block >= 0: + return block + if db_controller is None: + from simplyblock_core.db_controller import DBController + db_controller = DBController() + block = db_controller.claim_cntlid_block(snode, constants.MULTI_ATTACH_MAX_NODES) + snode.cntlid_block = block + return block + + +def subsystem_create_kwargs(snode, device, db_controller=None) -> dict: + """Extra ``rpc_client.subsystem_create`` kwargs for a device subsystem: + the node's controller-ID range for a multi-attach device, nothing + otherwise (the regular min_cntlid=1 default stays).""" + if not is_multi_attach(device): + return {} + lo, hi = cntlid_range(ensure_cntlid_block(snode, db_controller)) + return {"min_cntlid": lo, "max_cntlid": hi} + + +def attach_controller_kwargs(device) -> dict: + """Per-controller hold timeouts for a peer's controller to a multi-attach + device (``bdev_nvme_attach_controller``); nothing otherwise, so the + cluster-wide bdev_nvme options apply as before.""" + if not is_multi_attach(device): + return {} + return { + "ctrlr_loss_timeout_sec": constants.MULTI_ATTACH_HOLD_SEC, + "reconnect_delay_sec": constants.MULTI_ATTACH_RECONNECT_DELAY_SEC, + "fast_io_fail_timeout_sec": constants.MULTI_ATTACH_FAST_IO_FAIL_SEC, + } + + +def reservation_key(node) -> int: + """The persistent reservation key of ``node``'s alceml instances on array + LUNs: a node id unique in the cluster in the upper 32 bits (its + controller-ID block + 1, never 0); the instance sets the lower 32 bits + (its activation epoch). 0 when the node has no block yet.""" + block = getattr(node, "cntlid_block", -1) + if not isinstance(block, int) or block < 0: + return 0 + return (block + 1) << 32 + + +def reservations_supported(cluster, device) -> bool: + """Whether a new LUN device gets reservations (NVMeDevice.lun_reservations): + MULTI_ATTACH_RESERVATIONS on, the cluster's array honours reservations + (plugin capability supports_reservations), and the device's inline + checksum is not in fallback mode (the data plane refuses a key with + checksum_validation_method 2).""" + if not (constants.MULTI_ATTACH_RESERVATIONS and is_multi_attach(device)): + return False + from simplyblock_core import array, utils + if utils.alceml_checksum_params(cluster, device)[0] == 2: + return False + if not getattr(cluster, "array_plugin", ""): + return False + try: + caps = array.cached_plugin_for_cluster(cluster).capabilities() or {} + except Exception as e: # noqa: BLE001 + logger.warning("Array capabilities unavailable (%s); LUN %s without reservations", + e, getattr(device, "lun_wwn", "")) + return False + return bool(caps.get("supports_reservations")) + + +def reservations_enabled(cluster, device) -> bool: + """Whether the alceml of ``device`` runs with a reservation key: as + decided when it was marked a LUN (NVMeDevice.lun_reservations), the same + for every open of it.""" + return is_multi_attach(device) and getattr(device, "lun_reservations", False) is True + + +def warm_standby_enabled(cluster, device) -> bool: + """Whether ``device`` gets warm standby alcemls on its attached + non-serving nodes (lun_takeover.ensure_warm_standbys): a multi-attach + cluster with ``multi_attach_warm_standby`` set, and a device whose + alceml instances run with reservation keys (a standby activates by + preempting the old holder's reservation; without one nothing would fence + the old holder on activation).""" + return (is_multi_attach(cluster) + and getattr(cluster, "multi_attach_warm_standby", False) is True + and reservations_enabled(cluster, device)) + + +def alceml_create_kwargs(device, node=None, cluster=None) -> dict: + """Extra ``bdev_alceml_create`` kwargs for a multi-attach device's alceml + on ``node``: the stuck-I/O timeout, and the reservation key when + reservations are enabled (reservations_enabled); nothing otherwise.""" + if not is_multi_attach(device): + return {} + out = {"stuck_io_timeout_ms": constants.MULTI_ATTACH_ALCEML_STUCK_IO_MS} + if node is not None and cluster is not None and reservations_enabled(cluster, device): + key = reservation_key(node) + if key: + out["reservation_key"] = key + return out + + +def mark_lun_devices(cluster, snode, devices, blk_entries) -> None: + """Flag the lblk devices of a multi-attach cluster as array LUNs: set + ``multi_attach``, the LUN WWN (the inventory's WWN, else its serial, which + lsblk falls back to the WWN for) and the owner's ``node_paths`` entry. + No-op on any other cluster. An entry configured as the journal is left + unflagged; a device the add-node path later picks as the journal keeps + the flag but never uses it (the journal stack does not read it).""" + if not is_multi_attach(cluster): + return + by_serial = {e.get("serial"): e for e in (blk_entries or [])} + for dev in devices: + entry = by_serial.get(dev.serial_number, {}) + if entry.get("journal"): + continue + dev.multi_attach = True + dev.lun_reservations = reservations_supported(cluster, dev) + # A LUN the array plugin created carries the identity the plugin + # reported for it (array_controller.provision_node_lun); it is one of + # the WWN / serial the host reports, so lun_keys matches either way. + dev.lun_wwn = entry.get("lun_wwn") or entry.get("wwn") or dev.serial_number or "" + if entry.get("array_lun_id"): + dev.array_lun_id = entry["array_lun_id"] + dev.node_paths = { + snode.get_id(): { + "path": dev.by_id_path or dev.device_path, + "bdev": dev.nvme_bdev, + } + } + + +def own_multi_attach_devices(node, statuses) -> list: + """The node's own multi-attach data devices in one of ``statuses``: the + devices its distribs reach through a loopback ``remote_alceml_n1`` + controller instead of the local alceml.""" + return [d for d in (getattr(node, "nvme_devices", None) or []) + if is_multi_attach(d) and d.status in statuses and d.alceml_bdev] + + +# ------------------------------------------------------------------ phase 3 +# LUN takeover (controllers/lun_takeover) and the paths it touches: a device +# may now be served by a node other than its owner (serving_node_id). + +_NODE_ONLINE = "online" + + +def served_elsewhere(device, node_id) -> bool: + """A multi-attach device served by a node other than ``node_id``: that + node must not build the device's alceml, subsystem or anything above the + base bdev, nor report the device lost when it goes away.""" + return is_multi_attach(device) and device.serving_node() != node_id + + +def fenced_from(device, node_id) -> bool: + """A takeover fenced ``node_id`` from the LUN of ``device`` at the array + and nothing unfenced it since.""" + return is_multi_attach(device) and node_id in (getattr(device, "fenced_node_ids", None) or []) + + +def not_servable_here(device, node_id) -> bool: + """``node_id`` must not build a stack for ``device``: another node serves + its LUN, or ``node_id`` is fenced from it (the takeover after its failure + did not complete, or it has not been reintegrated yet -- + lun_takeover.reintegrate_fenced_node / unfence_node).""" + return served_elsewhere(device, node_id) or fenced_from(device, node_id) + + +def array_host(node) -> str: + """A storage node's identity on the storage array: its hostname, without + the ``_`` suffix a containerised node's hostname carries.""" + host = getattr(node, "hostname", "") or "" + suffix = f"_{getattr(node, 'rpc_port', '')}" + if suffix != "_" and host.endswith(suffix): + host = host[: -len(suffix)] + return host + + +def array_host_ref(node) -> str: + """How the array plugin names a node's host: the id ``ensure_host`` + returned (``StorageNode.array_host_id``; the EC2 instance id on EBS), + else the hostname (``array_host``).""" + return getattr(node, "array_host_id", "") or array_host(node) + + +def array_lun_ref(device) -> str: + """How the array plugin names a device's LUN: the id the plugin created + it under (``NVMeDevice.array_lun_id``), else its WWN.""" + return getattr(device, "array_lun_id", "") or getattr(device, "lun_wwn", "") or "" + + +def standby_candidates(owner, failed_node_id) -> list[str]: + """Node ids that may take over ``owner``'s LUNs, in preference order: the + owner's LVS secondary, its tertiary, then the owner itself (when it is + alive while the node serving its LUN fails). The failed node is never a + candidate.""" + out: list[str] = [] + for nid in (getattr(owner, "secondary_node_id", ""), getattr(owner, "tertiary_node_id", ""), + owner.get_id()): + if nid and nid != failed_node_id and nid not in out: + out.append(nid) + return out + + +def devices_served_by(nodes, node_id) -> list: + """``(owner, device)`` for every multi-attach device of ``nodes`` whose + serving node is ``node_id`` (its own LUNs it still serves, and LUNs it took + over).""" + return [(owner, dev) for owner in nodes for dev in (owner.nvme_devices or []) + if is_multi_attach(dev) and dev.serving_node() == node_id] + + +def served_by_online_node(owner, nodes_by_id) -> list: + """``owner``'s multi-attach devices served by another node that is + ONLINE: reachable although the owner itself is down.""" + out = [] + for dev in getattr(owner, "nvme_devices", None) or []: + if not served_elsewhere(dev, owner.get_id()): + continue + server = nodes_by_id.get(dev.serving_node()) + if server is not None and server.status == _NODE_ONLINE: + out.append(dev) + return out + + +#: Device states in which a warm standby is kept (any other: removed). +_WARM_STANDBY_DEVICE_STATES = ("online",) +#: Device states whose LUN still counts in a node's warm-standby memory +#: budget (a LUN unavailable for a while gets its standby back). +_WARM_STANDBY_BUDGET_STATES = ("online", "unavailable", "read_only", "cannot_allocate") + + +def warm_standby_devices(cluster, node_id, nodes, statuses=_WARM_STANDBY_DEVICE_STATES) -> list: + """``(owner, device)`` of every LUN ``node_id`` keeps a warm standby of + (Cluster.multi_attach_warm_standby): ``warm_standby_enabled`` for the + device, the device in one of ``statuses``, the node among the LUN's + attached hosts (array_controller.desired_hosts: the owner, its LVS + secondary and tertiary), not the node serving it and not fenced from it. + Empty on any other cluster.""" + if not (is_multi_attach(cluster) and getattr(cluster, "multi_attach_warm_standby", False) is True): + return [] + from simplyblock_core.controllers.array_controller import desired_hosts + by_id = {n.get_id(): n for n in nodes} + out = [] + for owner in nodes: + for dev in getattr(owner, "nvme_devices", None) or []: + if not warm_standby_enabled(cluster, dev) or dev.status not in statuses: + continue + if dev.serving_node() == node_id or fenced_from(dev, node_id): + continue + if node_id in desired_hosts(owner, dev, by_id): + out.append((owner, dev)) + return out + + +def warm_standby_cluster(cluster) -> bool: + """A multi-attach cluster created with warm standbys.""" + return is_multi_attach(cluster) and getattr(cluster, "multi_attach_warm_standby", False) is True + + +def warm_standby_capacity(cluster, node_id, nodes) -> int: + """Raw capacity (bytes) of the LUNs ``node_id`` keeps (or will keep, once + they are back online) a warm standby of: the capacity whose alceml page + map the node holds on top of its own devices. 0 on any other cluster.""" + return sum(max(int(dev.size or 0), 0) for _, dev in + warm_standby_devices(cluster, node_id, nodes, _WARM_STANDBY_BUDGET_STATES)) + + +def normalize_wwn(value) -> str: + # lsblk reports a SCSI WWN as 0x, arrays as naa.; the same id. + v = (value or "").strip().lower() + return v.removeprefix("0x").removeprefix("naa.") + + +def lun_keys(device) -> set: + """Identities a host may report for the LUN of ``device``: its WWN and + its serial (lsblk falls back to the WWN for the serial).""" + return {k for k in (normalize_wwn(getattr(device, "lun_wwn", "")), + normalize_wwn(getattr(device, "serial_number", ""))) if k} + + +def foreign_lun_keys(nodes, node_id) -> set: + """LUN identities of the multi-attach devices of every node but + ``node_id``. A device with one of them, found on ``node_id``'s host, is + another node's LUN (all LUNs are presented to every node): it is never a + new device of ``node_id`` and is never formatted.""" + keys: set = set() + for node in nodes: + if node.get_id() == node_id: + continue + for dev in node.nvme_devices or []: + if is_multi_attach(dev): + keys |= lun_keys(dev) + return keys + + +def is_foreign_lun(device, foreign_keys) -> bool: + return bool(foreign_keys) and bool(lun_keys(device) & foreign_keys) + + +def pick_lun_path(entries, device): + """The path to open the LUN of ``device`` with on a takeover node, from + the node agent's ``lun_info`` answer. Returns ``(path, "")`` or + ``(None, reason)``. + + Only disks whose block 0 carries the alceml signature (read just now) and + that are at least as large as the device qualify. The by-id path the owner + opened it with is preferred when the node has it too (a WWN-derived by-id + name is the same on every host); otherwise exactly one qualifying disk is + required -- several host paths to one LUN are not chosen between.""" + if not entries: + return None, "the LUN is not visible on this node" + readable = [e for e in entries if e.get("has_alceml_signature") is True] + if not readable: + states = sorted({str(e.get("has_alceml_signature")) for e in entries}) + return None, f"no path to the LUN carries the alceml signature (signature: {', '.join(states)})" + size = int(getattr(device, "size", 0) or 0) + sized = [e for e in readable if int(e.get("size") or 0) >= size] + if not sized: + return None, f"the LUN is smaller here than the device ({size} bytes)" + owner_path = ((getattr(device, "node_paths", None) or {}).get(device.node_id, {}).get("path") + or getattr(device, "by_id_path", "") or "") + if owner_path.startswith("/dev/disk/by-id/"): + for e in sized: + if owner_path in (e.get("by_id_paths") or []) or owner_path == e.get("by_id_path"): + return owner_path, "" + if len(sized) == 1: + e = sized[0] + return e.get("by_id_path") or e.get("device_path"), "" + return None, f"{len(sized)} disks carry the LUN and none has the owner's path {owner_path!r}" diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index cdfa02aa69..30119716b1 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -343,6 +343,16 @@ def evict_cached_session(host, port) -> None: _session_pool.evict(host, port) + +def _add_affinity_floor(params: dict, affinity_floor) -> None: + """Add the placement anti-affinity floor ("device" or "relaxed", see + simplyblock_core.floors.migration_affinity_floor) to a migration start, + but only while constants.DISTR_AFFINITY_FLOOR_RPC is on: an older data + plane rejects an RPC with a parameter it does not know.""" + if affinity_floor and constants.DISTR_AFFINITY_FLOOR_RPC: + params["affinity_floor"] = affinity_floor + + class RPCClient: # ref: https://spdk.io/doc/jsonrpc.html @@ -467,7 +477,8 @@ def subsystem_get(self, nqn: str) -> dict | None: def subsystem_delete(self, nqn): return self._request("nvmf_delete_subsystem", nqn=nqn) - def subsystem_create(self, nqn, serial_number, model_number, min_cntlid=1, max_namespaces=32, allow_any_host=True): + def subsystem_create(self, nqn, serial_number, model_number, min_cntlid=1, max_namespaces=32, allow_any_host=True, + max_cntlid=None): params = { "nqn": nqn, "serial_number": serial_number, @@ -476,6 +487,10 @@ def subsystem_create(self, nqn, serial_number, model_number, min_cntlid=1, max_n "ana_reporting": True, "max_namespaces": max_namespaces, "model_number": model_number} + # Sent only when given (multi-attach device subsystems: a per-node + # controller-ID range), so every other call is unchanged. + if max_cntlid is not None: + params["max_cntlid"] = int(max_cntlid) return self._request("nvmf_create_subsystem", **params) def keyring_file_add_key(self, name: str, path: str, *, allow_existing: bool = False): @@ -965,7 +980,8 @@ def bdev_alceml_create(self, alceml_name, nvme_name, uuid, pba_init_mode=3, alceml_cpu_mask="", alceml_worker_cpu_mask="", pba_page_size=2097152, write_protection=False, full_page_unmap=False, checksum_method=0, cache_size=0, cache_eviction_threshold=0, - force_4k_atomic=False): + force_4k_atomic=False, stuck_io_timeout_ms=0, + reservation_key=0, standby=False, refresh_interval_ms=0): params = { "name": alceml_name, "cntr_path": nvme_name, @@ -1002,15 +1018,42 @@ def bdev_alceml_create(self, alceml_name, nvme_name, uuid, pba_init_mode=3, # atomicity (cluster.atomic_4k). Tell the data plane to skip its >=4K # block-size gate for fallback-mode checksum validation. params["cv_ignore_block_size"] = bool(force_4k_atomic) + # Fail pending I/O and unregister after this long without completion + # (data-plane default 4000 ms). Sent only when non-zero: multi-attach + # devices, whose array LUN needs longer than multipath failover. + if stuck_io_timeout_ms: + params["stuck_io_timeout_ms"] = int(stuck_io_timeout_ms) + # LUN failover (multi-attach): a non-zero key makes the instance + # register it on the LUN at open, and an active one (standby False) + # take the reservation, preempting the old holder (fence). Sent only + # when set: older data planes reject the parameters. + if reservation_key: + params["reservation_key"] = int(reservation_key) + if standby: + params["standby"] = True + # Warm standby: how often its page map is refreshed from the + # LUN's region counters (data-plane default 2000 ms when unset). + if refresh_interval_ms: + params["refresh_interval_ms"] = int(refresh_interval_ms) return self._request("bdev_alceml_create", **params) + + def alceml_get_role(self, name): + """LUN failover state of an alceml (role, state, epoch, keys, ...).""" + return self._request("alceml_get_role", name=name) def bdev_distrib_create(self, name, vuid, ndcs, npcs, num_blocks, block_size, jm_names, chunk_size, ha_comm_addrs=None, ha_inode_self=None, pba_page_size=2097152, distrib_cpu_mask="", ha_is_non_leader=True, jm_vuid=0, write_protection=False, full_page_unmap=True, shared_placement=False, write_protection_v2=False, page_copy=False, - synchronous_replication_mode=None, jm_n_local=None): + synchronous_replication_mode=None, jm_n_local=None, + allow_device_colocation: bool | None = None): """" + // Optional (cluster.raid_devices): several chunks of a stripe may share one + // device when no other device is writable. Sent only while + // constants.DISTR_AFFINITY_FLOOR_RPC is on (older data planes reject it). + "allow_device_colocation": true + // Optional, sync replication only: 0 disabled, 1 full. Not sent when None. "synchronous_replication_mode": 1 @@ -1072,6 +1115,8 @@ def bdev_distrib_create(self, name, vuid, ndcs, npcs, num_blocks, block_size, jm params["jm_n_local"] = jm_n_local if page_copy: params["page_copy"] = True + if allow_device_colocation is not None and constants.DISTR_AFFINITY_FLOOR_RPC: + params["allow_device_colocation"] = bool(allow_device_colocation) return self._request("bdev_distrib_create", **params) def distr_shared_placement(self, name=None, enable=True): @@ -1503,7 +1548,9 @@ def bdev_aio_create(self, name, filename, block_size=0): """Create an SPDK AIO bdev over a Linux block device (lblk cluster mode). ``filename`` is the device path — prefer the stable /dev/disk/by-id symlink. ``block_size`` 0 lets SPDK use the device's - logical block size.""" + logical block size. Callers go through utils.create_aio_bdev, which + passes the block size a device was formatted with + (NVMeDevice.aio_block_size).""" kwargs = {"name": name, "filename": filename} if block_size: kwargs["block_size"] = block_size @@ -1516,6 +1563,23 @@ def bdev_aio_rescan(self, name): """Re-read the backing device's size (device grow pickup).""" return self._request("bdev_aio_rescan", name=name) + def bdev_uring_create(self, name, filename, block_size=0): + """Create an SPDK io_uring bdev over a Linux block device (lblk + cluster mode, device-mapper nodes: see constants.LBLK_IO_ENGINE_URING). + Same arguments as bdev_aio_create; callers go through + utils.create_aio_bdev with io_engine="uring".""" + kwargs = {"name": name, "filename": filename} + if block_size: + kwargs["block_size"] = block_size + return self._request("bdev_uring_create", **kwargs) + + def bdev_uring_delete(self, name): + return self._request("bdev_uring_delete", name=name) + + def bdev_uring_rescan(self, name): + """Re-read the backing device's size (device grow pickup).""" + return self._request("bdev_uring_rescan", name=name) + def alceml_bdev_backend_resize_begin(self, name, request_timeout=None): """Put an alceml bdev into its resize state: new I/O is held, pending I/O drains, and the backing bdev is closed. Answers once the backing @@ -1610,7 +1674,7 @@ def distr_migration_status(self, name): return self._request("distr_migration_status", name=name) def distr_migration_failure_start(self, name, storage_ID, qos_high_priority=False, job_size=constants.MIG_JOB_SIZE, jobs=constants.MIG_PARALLEL_JOBS, - relaxed_mode=False): + relaxed_mode=False, affinity_floor: str | None = None): params = { "name": name, "storage_ID": storage_ID, @@ -1623,10 +1687,11 @@ def distr_migration_failure_start(self, name, storage_ID, qos_high_priority=Fals params["jobs"] = jobs if relaxed_mode: params["relaxed_mode"] = True + _add_affinity_floor(params, affinity_floor) return self._request("distr_migration_failure_start", **params) def distr_migration_expansion_start(self, name, qos_high_priority=False, job_size=constants.MIG_JOB_SIZE, jobs=constants.MIG_PARALLEL_JOBS, - relaxed_mode=False, keep_placed=False): + relaxed_mode=False, keep_placed=False, affinity_floor: str | None = None): params = { "name": name, } @@ -1638,6 +1703,7 @@ def distr_migration_expansion_start(self, name, qos_high_priority=False, job_siz params["jobs"] = jobs if relaxed_mode: params["relaxed_mode"] = True + _add_affinity_floor(params, affinity_floor) # Page copy distribs only (older data planes reject the parameter): a balancing migration keeps # columns that page copies placed on another online device with anti-affinity kept. if keep_placed: diff --git a/simplyblock_core/services/__main__.py b/simplyblock_core/services/__main__.py index e7637f5014..86c5e6c3ee 100644 --- a/simplyblock_core/services/__main__.py +++ b/simplyblock_core/services/__main__.py @@ -23,6 +23,7 @@ # advertise names that do nothing when run. _LIBRARIES = frozenset({ "hub_controller_manager", + "lun_watchdog", # a thread inside storage_node_monitor "migration_task_common", "replication_final_step", "task_runner_base", diff --git a/simplyblock_core/services/capacity_and_stats_collector.py b/simplyblock_core/services/capacity_and_stats_collector.py index c19d240059..6f1b49f27d 100644 --- a/simplyblock_core/services/capacity_and_stats_collector.py +++ b/simplyblock_core/services/capacity_and_stats_collector.py @@ -1,7 +1,7 @@ import statistics import time -from simplyblock_core import constants, db_controller, utils +from simplyblock_core import constants, db_controller, multi_attach, utils from simplyblock_core.controllers import device_events from simplyblock_core.models.nvme_device import NVMeDevice from simplyblock_core.models.stats import ( @@ -158,6 +158,13 @@ def add_device_stats(cl, device, capacity_dict, stats_dict, tick_rate=0): # size_prov and with the lvol/snapshot figures, which are effective by # construction. The raw numbers are kept in the *_raw fields. size_total_raw = int(capacity_dict['npages_nmax']*capacity_dict['pba_page_size']) + # alceml spans its whole block device (npages_nmax follows the backend + # size). After a LUN grew by more than the cluster-wide growth factor, + # only ``size`` of it is usable: placement fills every device in + # proportion to its frozen weight, so the rest is never reached. + backend_size, dev_size = getattr(device, "backend_size", 0), getattr(device, "size", 0) + if isinstance(backend_size, int) and isinstance(dev_size, int) and backend_size > dev_size > 0: + size_total_raw = size_total_raw * dev_size // backend_size size_used_raw = int(capacity_dict['npages_used']*capacity_dict['pba_page_size']) size_free_raw = size_total_raw - size_used_raw @@ -357,6 +364,22 @@ def add_cluster_stats(cl, records): +def device_stats_source(device, owner, owner_rpc, nodes_by_id): + """``(rpc_client, base_bdev)`` to read the capacity and I/O stats of + ``device`` from: its owner, or for a multi-attach device whose LUN + another node took over (lun_takeover), the serving node, where its + alceml runs now, with the base bdev the takeover recorded there. None + while that node is not online.""" + if not multi_attach.served_elsewhere(device, owner.get_id()): + return owner_rpc, device.nvme_bdev + host = nodes_by_id.get(device.serving_node()) + if host is None or host.status != StorageNode.STATUS_ONLINE: + return None + path = (device.node_paths or {}).get(host.get_id()) or {} + base = path.get("bdev") or utils.aio_bdev_name_for_serial(device.serial_number) + return host.rpc_client(timeout=5, retry=2), base + + # get DB controller db = db_controller.DBController() @@ -377,6 +400,7 @@ def main(): logger.error(f"Cluster has no storage nodes: {cl.get_id()}") all_lvols = db.get_mini_lvols() + nodes_by_id = {n.get_id(): n for n in snodes} node_records = [] cluster_device_records = [] for node in snodes: @@ -396,13 +420,19 @@ def main(): if device.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE]: logger.info(f"Device is skipped: {device.get_id()} status: {device.status}") continue + source = device_stats_source(device, node, rpc_client, nodes_by_id) + if source is None: + logger.info(f"Device is skipped: {device.get_id()} its serving node " + f"{device.serving_node()} is not online") + continue + dev_rpc, base_bdev = source try: - capacity_dict = rpc_client.alceml_get_capacity(device.alceml_name) + capacity_dict = dev_rpc.alceml_get_capacity(device.alceml_name) except Exception as e: logger.error(e) continue try: - ret = rpc_client.get_lvol_stats(device.nvme_bdev) + ret = dev_rpc.get_lvol_stats(base_bdev) except RPCException as e: logger.error("Failed to get iostat for device %s: %s", device.get_id(), e) continue diff --git a/simplyblock_core/services/device_monitor.py b/simplyblock_core/services/device_monitor.py index c0695401f4..89f17f39fc 100644 --- a/simplyblock_core/services/device_monitor.py +++ b/simplyblock_core/services/device_monitor.py @@ -1,6 +1,6 @@ import time -from simplyblock_core import constants, db_controller, utils +from simplyblock_core import constants, db_controller, multi_attach, utils from simplyblock_core.controllers import device_controller, tasks_controller from simplyblock_core.controllers.device_controller import CAUSE_LOCAL_FAILURE from simplyblock_core.models.cluster import Cluster @@ -46,7 +46,7 @@ def _check_aio_hung_io(node, rpc_client) -> list: into device failures (cf. constants NVME_TIMEOUT_US rationale).""" stalled = [] for dev in node.nvme_devices: - if dev.bdev_type != "aio" or dev.status != NVMeDevice.STATUS_ONLINE: + if dev.bdev_type != "aio" or dev.status != NVMeDevice.STATUS_ONLINE or _not_local(dev, node): _aio_progress.pop(dev.get_id(), None) continue try: @@ -82,6 +82,17 @@ def _check_aio_hung_io(node, rpc_client) -> list: return stalled +def _not_local(dev, node) -> bool: + """Multi-attach: the device's LUN is not served from ``node`` (a takeover + moved it, or ``node`` is fenced from it). The node holds no stack for it + and its host may not even see the LUN (the fence detached it), so neither + a missing disk nor a missing stack there says anything about the device. + 2026-11-19 FTT1 bed: after an SPDK kill and takeover, the fenced owner's + detached LUN was swept as hot-removed; device_remove of a 1+0 device + turned into read errors on every lvstore and a cluster suspension.""" + return multi_attach.not_servable_here(dev, node.get_id()) + + def _check_aio_device_presence(node) -> list: """Return the node's aio devices whose backing block device has been absent from the host inventory for AIO_DEVICE_ABSENT_POLLS consecutive @@ -89,7 +100,8 @@ def _check_aio_device_presence(node) -> list: aio_devs = [dev for dev in node.nvme_devices if dev.bdev_type == "aio" and dev.status in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE, - NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE]] + NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE] + and not _not_local(dev, node)] if not aio_devs: return [] try: diff --git a/simplyblock_core/services/health_check_service.py b/simplyblock_core/services/health_check_service.py index 73ecd6ee80..82d961667e 100644 --- a/simplyblock_core/services/health_check_service.py +++ b/simplyblock_core/services/health_check_service.py @@ -3,8 +3,15 @@ import time from datetime import datetime -from simplyblock_core import constants, db_controller, storage_node_ops, utils +from simplyblock_core import ( + constants, + db_controller, + multi_attach, + storage_node_ops, + utils, +) from simplyblock_core.controllers import ( + array_controller, device_controller, device_events, health_controller, @@ -292,6 +299,11 @@ def _group_devices(snode=snode): if device.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE]: logger.info(f"Device skipped: {device.get_id()} status: {device.status}") continue + if multi_attach.served_elsewhere(device, snode.get_id()): + # Multi-attach: another node took the LUN over; this node + # has no stack for it (its loopback is checked below). + logger.info(f"Device skipped: {device.get_id()} served by {device.serving_node()}") + continue passed = True @@ -350,7 +362,11 @@ def _group_devices(snode=snode): for remote_device in snode.remote_devices: org_dev = db.get_storage_device_by_id(remote_device.get_id()) - org_node = db.get_storage_node_by_id(remote_device.node_id) + # The node exporting the device: its owner, or the node a + # multi-attach takeover moved its LUN to. + org_node = db.get_storage_node_by_id( + org_dev.serving_node() if multi_attach.is_multi_attach(org_dev) + else remote_device.node_id) # Only treat a missing remote device as a fault when the owning node # is ONLINE/DOWN/UNREACHABLE. If the owner is mid-transition (restart, # shutdown, ...) the connection is expected to be gone — skip it. @@ -361,7 +377,9 @@ def _group_devices(snode=snode): # connect. _peer_connections_relevant also admits # UNREACHABLE, which is fine for judging whether a missing # connection is a fault and wrong for deciding to dial out. - if org_dev.nvmf_multipath: + # Multi-attach: also single-path, the controller may + # still miss the path to the node that took it over. + if org_dev.nvmf_multipath or multi_attach.is_multi_attach(org_dev): ctrl_name = f"remote_{org_dev.alceml_bdev}" if org_dev.alceml_bdev else None if ctrl_name and health_controller.repairs_allowed(org_node): device_repair_jobs.append((ctrl_name, org_dev, snode)) @@ -681,6 +699,13 @@ def main(): continue clusters = db.get_clusters() for cluster in clusters: + # Multi-attach cluster with an array: keep every LUN presented to + # its owner's current LVS secondary / tertiary (roles move on + # node removal and migration). No-op on every other cluster. + try: + array_controller.reconcile_in_background(cluster) + except Exception as e: + logger.error(f"Array LUN reconcile of {cluster.get_id()} not started: {e}") for node in db.get_storage_nodes_by_cluster_id(cluster.get_id()): node_id = node.get_id() if node_id not in threads_maps or threads_maps[node_id].is_alive() is False: diff --git a/simplyblock_core/services/lun_watchdog.py b/simplyblock_core/services/lun_watchdog.py new file mode 100644 index 0000000000..8d9f7756b4 --- /dev/null +++ b/simplyblock_core/services/lun_watchdog.py @@ -0,0 +1,331 @@ +"""Multi-attach clusters: fast loss detection of the nodes serving LUNs. + +With 1+0 erasure coding every volume has data on every device, so all of +them wait on a lost node's LUN until a standby has taken it over +(controllers/lun_takeover). The regular monitor declares a node lost only +after its ping/API checks have failed: 1 s for a killed SPDK, 10-20 s for a +host that is gone, and the OFFLINE verdict takes far longer. That is the +volumes' I/O stall. + +This watchdog probes every node that serves a LUN every +``MULTI_ATTACH_PROBE_INTERVAL_SEC``: a ``spdk_get_version`` RPC with +``MULTI_ATTACH_PROBE_TIMEOUT_SEC``. While that fails it also asks the node +agent whether SPDK runs. The node is lost + +* at once when the agent reports SPDK not running (killed or crashed); +* when neither SPDK nor the agent has answered for + ``MULTI_ATTACH_LOST_AFTER_SEC`` (host down or cut off); + +and never while the agent reports SPDK running: a slow RPC is the regular +monitor's case. A node that answers again within the window is not taken +over at all. + +The takeover of a lost node's LUNs starts straight away, each LUN in its own +thread (the monitor's ``_start_lun_takeover``, idempotent). It fences the +node at the array before anything else, so a node that is alive behind a +management network fault loses nothing; its LUNs pause for the takeover. + +Not when this process is the one cut off: no takeover while more nodes of a +cluster fail their probes than it tolerates (``max_fault_tolerance``), or +while no other node of it has answered recently. +""" + +from __future__ import annotations + +import threading +import time +from concurrent.futures import ThreadPoolExecutor + +from simplyblock_core import array, constants, multi_attach, utils +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode + +logger = utils.get_logger(__name__) + +#: Node states the probe watches. A node in a restart or shutdown is owned +#: by that flow; an OFFLINE one was handled when it went offline. +WATCH_STATUSES = (StorageNode.STATUS_ONLINE, StorageNode.STATUS_UNREACHABLE, + StorageNode.STATUS_DOWN, StorageNode.STATUS_SCHEDULABLE) + +#: Device states whose LUN a lost node takes down with it. +SERVING_STATES = (NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, + NVMeDevice.STATUS_CANNOT_ALLOCATE) + +#: Probe answers. +OK = "ok" # SPDK answered +SPDK_DOWN = "spdk_down" # SPDK did not; the agent says it is not running +SPDK_UP = "spdk_up" # SPDK did not; the agent says it runs (slow RPC) +NO_ANSWER = "no_answer" # neither answered + +#: Re-trigger of a node still lost (its takeover threads are idempotent). +REFIRE_SEC = 5.0 +#: How often the set of watched nodes is re-read. +REFRESH_SEC = 1.0 + + +def _rpc_answers(node, timeout) -> bool: + try: + node.rpc_client(timeout=timeout, retry=0).get_version() + return True + except Exception: # noqa: BLE001 + return False + + +def _agent_says(node, timeout) -> str: + try: + is_up, _ = node.client(timeout=timeout, retry=0, connect_retry=0).spdk_process_is_up( + node.rpc_port, node.cluster_id) + except Exception: # noqa: BLE001 + return NO_ANSWER + if is_up is True: + return SPDK_UP + if is_up is False: + return SPDK_DOWN + return NO_ANSWER + + +def probe(node, failing=False, timeout=None) -> str: + """One probe of ``node``. While it is already ``failing`` the RPC and the + agent are asked at the same time, else the agent only after the RPC + failed.""" + timeout = timeout or constants.MULTI_ATTACH_PROBE_TIMEOUT_SEC + if not failing: + if _rpc_answers(node, timeout): + return OK + return _agent_says(node, timeout) + with ThreadPoolExecutor(max_workers=2) as pool: + rpc = pool.submit(_rpc_answers, node, timeout) + agent = pool.submit(_agent_says, node, timeout) + if rpc.result(): + return OK + return agent.result() + + +class _Track: + """Probe history of one node.""" + + __slots__ = ("answer", "fired_at", "first_fail", "last_ok") + + def __init__(self, now): + self.first_fail = None + self.last_ok = now + self.fired_at = None + self.answer = OK + + +class Watchdog: + """The decisions, apart from the threads that probe (testable).""" + + def __init__(self, start_takeover, db=None): + self.start_takeover = start_takeover + self.db = db + self.lock = threading.Lock() + self.tracks: dict[str, _Track] = {} + self.cluster_of: dict[str, str] = {} + self.warmed: dict[str, float] = {} + + def _db(self): + if self.db is None: + self.db = DBController() + return self.db + + # ----------------------------------------------------------- watched + + def watched(self) -> dict: + """``{node_id: node}`` of the nodes to probe now: in a multi-attach + cluster that is not in activation, in a WATCH_STATUSES state, serving + at least one LUN in a SERVING_STATES state.""" + out = {} + db = self._db() + for cluster in db.get_clusters(): + if not multi_attach.is_multi_attach(cluster): + continue + if cluster.status == Cluster.STATUS_IN_ACTIVATION: + continue + self._warm(cluster) + nodes = db.get_storage_nodes_by_cluster_id(cluster.get_id()) + for node in nodes: + if node.status not in WATCH_STATUSES: + continue + if any(dev.status in SERVING_STATES + for _o, dev in multi_attach.devices_served_by(nodes, node.get_id())): + out[node.get_id()] = node + return out + + def _warm(self, cluster): + """Keep the cluster's array connection ready for a fence (connecting + costs about a second), refreshed at half its cache time.""" + cid = cluster.get_id() + now = time.monotonic() + half = constants.ARRAY_PLUGIN_CACHE_SEC / 2 + if not getattr(cluster, "array_plugin", "") or now - self.warmed.get(cid, -1e9) < half: + return + self.warmed[cid] = now + + def _connect(): + try: + array.cached_plugin_for_cluster(cluster, max_age=half) + except Exception as e: # noqa: BLE001 + logger.warning("LUN watchdog: array connection of %s not made: %s", cid, e) + self.warmed[cid] = time.monotonic() - half + 30 # again in 30 s + threading.Thread(target=_connect, name=f"array-warm-{cid[:8]}", daemon=True).start() + + # ---------------------------------------------------------- decisions + + def record(self, node, answer, now, sent=None) -> str | None: + """Note a probe answer of ``node`` at ``now`` to a probe ``sent`` then + (default ``now``); the reason the node is lost, or None. The silence + counts from when the first unanswered probe was sent: a probe that + times out returns only after its timeouts.""" + nid = node.get_id() + with self.lock: + self.cluster_of[nid] = node.cluster_id + t = self.tracks.setdefault(nid, _Track(now)) + t.answer = answer + if answer == OK: + if t.first_fail is not None: + logger.info("LUN watchdog: %s answers again after %.1fs", + nid, now - t.first_fail) + t.first_fail, t.fired_at, t.last_ok = None, None, now + return None + if answer == SPDK_UP: + # The agent saw SPDK running: the node is there, its RPC is + # slow. The silence restarts at the next unanswered probe (it + # used to survive this answer, and one agent timeout after it + # read as "no answer for 6.1s", 2026-11-20 FTT2 bed). + t.first_fail = None + return None + if t.first_fail is None: + t.first_fail = now if sent is None else sent + if answer == SPDK_DOWN: + return "SPDK not running" + silent = now - t.first_fail + if silent >= constants.MULTI_ATTACH_LOST_AFTER_SEC: + return f"no answer for {silent:.1f}s" + return None + + def _observer_ok(self, cluster, nid, now) -> str: + """Why the takeover of ``nid`` must wait because this process may be + the one cut off; empty when it may go ahead.""" + cid = cluster.get_id() + with self.lock: + peers = {n: t for n, t in self.tracks.items() + if self.cluster_of.get(n) == cid} + failing = [n for n, t in peers.items() if t.first_fail is not None] + ftt = max(1, int(getattr(cluster, "max_fault_tolerance", 1) or 1)) + if len(failing) > ftt: + return (f"{len(failing)} nodes of the cluster fail their probes, more than it " + f"tolerates ({ftt}): this monitor may be the one cut off") + recent = constants.MULTI_ATTACH_LOST_AFTER_SEC + 2 * constants.MULTI_ATTACH_PROBE_TIMEOUT_SEC + if not any(n != nid and t.first_fail is None and now - t.last_ok <= recent + for n, t in peers.items()): + return "no other node of the cluster answered recently" + return "" + + def fire(self, node_id, reason, now) -> list: + """Start the takeover of every LUN ``node_id`` serves; the device ids + started. At most every REFIRE_SEC per node.""" + with self.lock: + t = self.tracks.get(node_id) + if t is None or (t.fired_at is not None and now - t.fired_at < REFIRE_SEC): + return [] + db = self._db() + node = db.get_storage_node_by_id(node_id) + if node.status not in WATCH_STATUSES: + return [] + cluster = db.get_cluster_by_id(node.cluster_id) + if not multi_attach.is_multi_attach(cluster): + return [] + why_not = self._observer_ok(cluster, node_id, now) + if why_not: + logger.warning("LUN watchdog: %s is lost (%s) but no takeover: %s", + node_id, reason, why_not) + return [] + devs = [dev for _o, dev in multi_attach.devices_served_by( + db.get_storage_nodes_by_cluster_id(node.cluster_id), node_id) + if dev.status in SERVING_STATES] + with self.lock: + t.fired_at = now + if not devs: + return [] + logger.warning("LUN watchdog: node %s lost (%s); taking over %d LUN(s): %s", + node_id, reason, len(devs), [d.get_id() for d in devs]) + started = [] + for dev in devs: + try: + self.start_takeover(dev.get_id(), node_id, f"node lost: {reason}") + started.append(dev.get_id()) + except Exception as e: # noqa: BLE001 + logger.error("LUN watchdog: takeover of %s not started: %s", dev.get_id(), e) + return started + + def forget(self, node_id): + with self.lock: + self.tracks.pop(node_id, None) + self.cluster_of.pop(node_id, None) + + # ------------------------------------------------------------ threads + + def _watch_node(self, node_id, nodes): + """Probe one node until it is no longer watched.""" + failing = False + while True: + node = nodes.get(node_id) + if node is None: + self.forget(node_id) + return + started = time.monotonic() + try: + answer = probe(node, failing) + now = time.monotonic() + failing = answer != OK + reason = self.record(node, answer, now, started) + if reason: + self.fire(node_id, reason, now) + except Exception as e: # noqa: BLE001 + logger.error("LUN watchdog: probe of %s failed: %s", node_id, e) + wait = started + constants.MULTI_ATTACH_PROBE_INTERVAL_SEC - time.monotonic() + if wait > 0: + time.sleep(wait) + + def run(self): + nodes: dict = {} + threads: dict[str, threading.Thread] = {} + while True: + try: + fresh = self.watched() + except Exception as e: # noqa: BLE001 + logger.error("LUN watchdog: watched nodes not read: %s", e) + fresh = None + if fresh is not None: + for nid in list(nodes): + if nid not in fresh: + nodes.pop(nid, None) + nodes.update(fresh) + for nid in fresh: + t = threads.get(nid) + if t is None or not t.is_alive(): + t = threading.Thread(target=self._watch_node, args=(nid, nodes), + name=f"lun-watch-{nid[:8]}", daemon=True) + threads[nid] = t + t.start() + for nid in [n for n, t in threads.items() if not t.is_alive()]: + threads.pop(nid, None) + time.sleep(REFRESH_SEC) + + +_started: Watchdog | None = None + + +def start(start_takeover) -> Watchdog: + """Start the watchdog thread of this process (once).""" + global _started + if _started is None: + _started = Watchdog(start_takeover) + threading.Thread(target=_started.run, name="lun-watchdog", daemon=True).start() + logger.info("LUN watchdog started (probe %.1fs, lost after %.1fs)", + constants.MULTI_ATTACH_PROBE_INTERVAL_SEC, + constants.MULTI_ATTACH_LOST_AFTER_SEC) + return _started diff --git a/simplyblock_core/services/main_distr_event_collector.py b/simplyblock_core/services/main_distr_event_collector.py index 5a92713601..947722dc58 100644 --- a/simplyblock_core/services/main_distr_event_collector.py +++ b/simplyblock_core/services/main_distr_event_collector.py @@ -10,6 +10,7 @@ constants, db_controller, distr_controller, + multi_attach, rpc_client, utils, ) @@ -274,6 +275,22 @@ def process_device_event(event, logger): event.status = f'late_by_{int(time_delta.total_seconds())}s_skipping' return + if multi_attach.served_elsewhere(device_obj, event_node_obj.get_id()): + # A multi-attach LUN this node reaches through another node's + # export: its I/O errors are about that path. The LUN watchdog + # and the takeover handle a lost serving node and switch every + # node's path; marking the device unavailable for this node (or + # dropping its remote device) only turns a short hold into + # failed I/O on 1+0, and nothing marks it available again + # (2026-11-21 FTT2 bed: such events, processed during a + # takeover, left a device unavailable in a node's map; an LVS + # failover read there failed and its SPDK aborted). + logger.info(f"Multi-attach device {device_obj.get_id()} is served by " + f"{device_obj.serving_node()}, not by reporting node {event_node_obj.get_id()}: " + f"{event.message} left to the LUN takeover") + event.status = 'skipped:multi_attach_remote_path' + return + if device_obj.is_connection_in_progress_to_node(event_node_obj.get_id()): logger.warning("Connection attempt was found from node to device, sleeping 5 seconds") time.sleep(5) @@ -399,7 +416,12 @@ def process_device_event(event, logger): return - if device_node_obj.get_id() == event_node_obj.get_id(): + # On multi-attach the home node of a LUN served by another node reads + # it remotely like any peer: its error is about that path, not the + # LUN (2026-11-20 FTT2 bed: a home node's error_read on the dead + # serving node's export, processed after the takeover, marked the + # taken-over LUN unavailable cluster-wide and blocked every LVS). + if device_node_obj.get_id() == event_node_obj.get_id() and not multi_attach.served_elsewhere(device_obj, event_node_obj.get_id()): # The device's home node itself reported an IO error against its # local device — this is the only path that may count toward the # per-device flap budget. The CAUSE_LOCAL_FAILURE opt-in is diff --git a/simplyblock_core/services/migration_task_common.py b/simplyblock_core/services/migration_task_common.py index 4cd8fa11f8..ff8e04d425 100644 --- a/simplyblock_core/services/migration_task_common.py +++ b/simplyblock_core/services/migration_task_common.py @@ -9,7 +9,7 @@ import errno from datetime import UTC, datetime -from simplyblock_core import db_controller, storage_node_ops, utils +from simplyblock_core import db_controller, floors, storage_node_ops, utils from simplyblock_core.controllers import ( activation_gate, sync_replication_controller, @@ -577,13 +577,26 @@ def needs_relaxed_placement(member_nodes, ndcs, npcs): """Whether a migration on a cluster of ``member_nodes`` must run relaxed. Strict placement puts every chunk of a stripe on a different node. A - 1- or 2-node cluster, or one with fewer nodes than chunks per stripe, - has no such placement, so a strict migration has nowhere to move data - and never completes; relocation then needs a third node. ultra already - relaxes writes in that case on its own, but migrations only relax when - asked (``relaxed_mode``, default false). + 1- or 2-node cluster, or one with no more nodes than chunks per stripe, + has no such placement with a node to move to: with EXACTLY ndcs+npcs + member nodes every node already holds a chunk of every stripe, so a + strict migration away from one of them has nowhere to put the chunk + and never completes (a dead end once a node is down -- hence ``<=``, not + ``<``). ultra already relaxes writes in that case on its own, but + migrations only relax when asked (``relaxed_mode``, default false). """ - return member_nodes <= 2 or member_nodes < ndcs + npcs + return member_nodes <= 2 or member_nodes <= ndcs + npcs + + +def migration_affinity_floor(cluster): + """The data plane's placement anti-affinity floor for a migration on + ``cluster`` (a cluster record or its id): ``"relaxed"`` when its data + devices are protected below simplyblock (``raid_devices``), ``"device"`` + otherwise. Sent as ``affinity_floor`` beside ``relaxed_mode`` once + constants.DISTR_AFFINITY_FLOOR_RPC is on.""" + if isinstance(cluster, str): + cluster = db.get_cluster_by_id(cluster) + return floors.migration_affinity_floor(cluster) def migration_relaxed(cluster_id): diff --git a/simplyblock_core/services/storage_node_monitor.py b/simplyblock_core/services/storage_node_monitor.py index 0d73802d63..e7a8e231e0 100644 --- a/simplyblock_core/services/storage_node_monitor.py +++ b/simplyblock_core/services/storage_node_monitor.py @@ -9,12 +9,15 @@ cluster_ops, constants, db_controller, + floors, + multi_attach, storage_network, storage_node_ops, utils, ) from simplyblock_core.controllers import ( activation_gate, + cluster_events, device_controller, health_controller, port_release_budget, @@ -657,14 +660,65 @@ def _count_status(cluster, snodes, jm_repl_by_node, dp_quorum_by_node, cluster_t online_nodes, online_devices, jm_replication_tasks) +# cluster id -> the floor violations last reported for it (the event and the +# warning are emitted when the set changes, not on every 3s tick). +_floor_violations_reported: dict = {} + + +def _apply_floors(cluster, snodes, status): + """DEGRADED, naming the floor, while the ONLINE resources are below a + floor (simplyblock_core.floors) and the existing rules did not already + SUSPEND the cluster. The suspension rules are untouched: an outage may + take a running cluster below the floors, nothing is suspended for that + alone -- the data plane keeps serving and migrating and the rebalance + restores the protection. The floor is recorded as a cluster event and a + warning whenever the set of violated floors changes.""" + violations = floors.evaluate(cluster, snodes) + key = cluster.get_id() + previous = _floor_violations_reported.get(key) + if violations != previous: + if violations: + logger.warning("Cluster %s below resource floor: %s", key, "; ".join(violations)) + try: + cluster_events.cluster_floor_violation(cluster, violations) + except Exception as e: # noqa: BLE001 - the status verdict must not fail on an event + logger.debug("floor event not recorded: %s", e) + elif previous: + logger.info("Cluster %s back at or above every resource floor", key) + try: + cluster_events.cluster_floor_restored(cluster) + except Exception as e: # noqa: BLE001 + logger.debug("floor event not recorded: %s", e) + _floor_violations_reported[key] = violations + if violations and status != Cluster.STATUS_SUSPENDED: + return Cluster.STATUS_DEGRADED + return status + + def _status_verdict(cluster, snodes, affected_physical_nodes, online_nodes, online_devices, jm_replication_tasks): - """The cluster status for one set of affected hosts.""" + """The cluster status for one set of affected hosts: the suspension / + degradation rules (``_status_verdict_rules``), then the floors.""" + status = _status_verdict_rules(cluster, snodes, affected_physical_nodes, online_nodes, + online_devices, jm_replication_tasks) + return _apply_floors(cluster, snodes, status) + + +def _status_verdict_rules(cluster, snodes, affected_physical_nodes, online_nodes, + online_devices, jm_replication_tasks): + """The suspension / degradation rules for one set of affected hosts.""" affected_nodes = len(affected_physical_nodes) # ndcs n = 2 # npcs k = 1 n = cluster.distr_ndcs k = cluster.distr_npcs + if getattr(cluster, "multi_attach", False) is True: + # Multi-attach array LUNs run 1+0: no parity, but a lost node's LUN is + # taken over by a standby (controllers/lun_takeover), so the cluster + # survives as many lost nodes as its FTT. With k = npcs = 0 the first + # node loss suspended the cluster, which shut the other nodes down + # and with them the takeover's standby (found on the multi-attach bed). + k = max(k, int(getattr(cluster, "max_fault_tolerance", 0) or 1)) # Failure-domain-aware suspend criteria. When the cluster was created with # --enable-failure-domain, chunks are spread across distinct domains, so it @@ -1143,10 +1197,7 @@ def _force_offline_lost_site_node(node): tasks_controller.cancel_pending_node_restart_tasks(node.cluster_id, node.get_id()) storage_node_ops.set_node_status(node.get_id(), StorageNode.STATUS_OFFLINE, caused_by="sync_site_lost") - for dev in node.nvme_devices: - if dev.status in (NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, - NVMeDevice.STATUS_CANNOT_ALLOCATE): - device_controller.device_set_unavailable(dev.get_id()) + _set_node_devices_unavailable(node, "site lost") def _drain_nodes(nodes, what, gone=None) -> bool: @@ -1843,6 +1894,86 @@ def _take_over_leaderless_lvstores(snode): lvs_name, leader.get_id(), snode.get_id(), snode.status) +#: Device states that are lost with their node (set unavailable). +_DEV_SERVING_STATES = (NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, + NVMeDevice.STATUS_CANNOT_ALLOCATE) + +#: device id -> running LUN takeover thread (multi-attach clusters). +lun_takeover_threads: dict[str, threading.Thread] = {} + + +def _lun_takeover_worker(device_id, failed_node_id, reason, started): + """Take over one LUN; mark the device unavailable when it was not taken + over (the regular node-loss behaviour). When the LUN was fenced but no + standby could read it, the peers keep holding I/O: the device is marked + unavailable only when their hold has run out.""" + from simplyblock_core.controllers import lun_takeover + try: + outcome = lun_takeover.take_over_lun_outcome(device_id, failed_node_id, reason) + except Exception as e: # noqa: BLE001 + logger.error("LUN takeover of %s raised: %s", device_id, e) + outcome = lun_takeover.FAILED + if outcome in (lun_takeover.TAKEN_OVER, lun_takeover.IN_PROGRESS): + return + if outcome == lun_takeover.NO_STANDBY: + wait = started + constants.MULTI_ATTACH_HOLD_SEC - time.monotonic() + if wait > 0: + time.sleep(wait) + try: + dev = db.get_storage_device_by_id(device_id) + except KeyError: + return + if dev.serving_node() != failed_node_id or dev.status not in _DEV_SERVING_STATES: + return + logger.warning("LUN of device %s not taken over (%s); setting it unavailable", + device_id, outcome) + device_controller.device_set_unavailable(device_id) + + +def _start_lun_takeover(device_id, failed_node_id, reason): + existing = lun_takeover_threads.get(device_id) + if existing is not None and existing.is_alive(): + logger.info("LUN takeover of %s already running", device_id) + return + t = threading.Thread(target=_lun_takeover_worker, + args=(device_id, failed_node_id, reason, time.monotonic()), + name=f"lun-takeover-{device_id[:8]}", daemon=True) + lun_takeover_threads[device_id] = t + t.start() + + +def _set_node_devices_unavailable(node, reason): + """The devices lost with ``node``: set unavailable, as always. + + Multi-attach clusters (only): a LUN the node served -- its own or one it + had taken over -- is taken over by a standby instead, each in its own + thread, started before anything else so the fence comes first and the + takeover fits in the peers' hold; a LUN of the node that another node + serves is not lost with it and is left alone. Only devices whose takeover + fails are set unavailable (by the takeover thread).""" + try: + cluster = db.get_cluster_by_id(node.cluster_id) + except Exception as e: # noqa: BLE001 + # Unknown cluster mode: the regular behaviour (never a takeover). + logger.warning("Cluster of %s not read (%s); devices set unavailable", node.get_id(), e) + cluster = None + if not multi_attach.is_multi_attach(cluster): + for dev in node.nvme_devices: + if dev.status in _DEV_SERVING_STATES: + # Default cause is correct: node going offline cascades + # to all its devices, that does not count as a flap. + device_controller.device_set_unavailable(dev.get_id()) + return + node_id = node.get_id() + for _owner, dev in multi_attach.devices_served_by( + db.get_storage_nodes_by_cluster_id(node.cluster_id), node_id): + if dev.status in _DEV_SERVING_STATES: + _start_lun_takeover(dev.get_id(), node_id, reason) + for dev in node.nvme_devices: + if dev.status in _DEV_SERVING_STATES and not multi_attach.is_multi_attach(dev): + device_controller.device_set_unavailable(dev.get_id()) + + def _ana_failover_rearm(node_id) -> None: """Forget the one-shot marker so the next offline episode re-triggers.""" _ana_failover_applied.discard(node_id) @@ -1868,12 +1999,9 @@ def set_node_offline(node): StorageNode.STATUS_RESTARTING]: try: storage_node_ops.set_node_status(node.get_id(), StorageNode.STATUS_OFFLINE) - for dev in node.nvme_devices: - if dev.status in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, - NVMeDevice.STATUS_CANNOT_ALLOCATE]: - # Default cause is correct: node going offline cascades - # to all its devices, that does not count as a flap. - device_controller.device_set_unavailable(dev.get_id()) + # Multi-attach: LUN takeovers start here, before the LVS ANA + # failover below (they run in parallel with it). + _set_node_devices_unavailable(node, "node offline") except Exception as e: logger.debug("Setting node to OFFLINE state failed") logger.error(e) @@ -1909,6 +2037,26 @@ def set_node_offline(node): logger.error("add_node_to_auto_restart for %s failed: %s", node.get_id(), ar_e) +def _take_over_luns_of_unreachable(node): + """Multi-attach clusters (only): start the takeover of the LUNs a node + serves as soon as it is UNREACHABLE. Usually the fast probe + (services/lun_watchdog) has started it seconds earlier; this is its + backstop. The takeover fences the node at the array first; each LUN gets + one thread (_start_lun_takeover is idempotent across monitor cycles).""" + node_id = node.get_id() + try: + cluster = db.get_cluster_by_id(node.cluster_id) + except Exception as e: # noqa: BLE001 + logger.warning("Cluster of %s not read (%s); no LUN takeover", node_id, e) + return + if not multi_attach.is_multi_attach(cluster): + return + for _owner, dev in multi_attach.devices_served_by( + db.get_storage_nodes_by_cluster_id(node.cluster_id), node_id): + if dev.status in _DEV_SERVING_STATES: + _start_lun_takeover(dev.get_id(), node_id, "node unreachable") + + def set_node_unreachable(node): # Same rationale as set_node_offline: when the runner owns the node's # status transitions (IN_SHUTDOWN → OFFLINE → IN_RESTART → ONLINE), the @@ -1923,6 +2071,11 @@ def set_node_unreachable(node): logger.debug("Setting node to UNREACHABLE state failed") logger.error(e) + try: + _take_over_luns_of_unreachable(node) + except Exception as e: # noqa: BLE001 + logger.error("LUN takeover trigger for unreachable node %s failed: %s", node.get_id(), e) + # Check data-plane health from surviving nodes. If all online peers # report the unreachable node's remote JM as disconnected, the data # plane is truly down and we can escalate to offline. @@ -2175,10 +2328,7 @@ def set_node_schedulable(node): # initiate shutdown # initiate restart # tasks_controller.add_node_to_auto_restart(node) - for dev in node.nvme_devices: - if dev.status in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, - NVMeDevice.STATUS_CANNOT_ALLOCATE]: - device_controller.device_set_unavailable(dev.get_id()) + _set_node_devices_unavailable(node, "node schedulable") update_cluster_status(node.cluster_id) except Exception as e: logger.debug("Setting node to SCHEDULABLE state failed") @@ -2207,6 +2357,60 @@ def node_rpc_timeout_check_and_report(node): return False +#: Node id -> (cluster id, monotonic time) of its last failed port check, for +#: the correlation guard below. Cleared by a passing check. +_port_failed_at: dict = {} +_port_failed_lock = threading.Lock() +#: A failed port check counts toward the correlation for this long: a few node +#: check cycles. +PORT_FAILURE_CORRELATION_SEC = 30.0 +_correlated_event_at: dict = {} + + +def _multi_attach_cluster(cluster_id) -> bool: + try: + return multi_attach.is_multi_attach(db.get_cluster_by_id(cluster_id)) + except Exception: # noqa: BLE001 + return False + + +def _note_port_check(snode, result): + with _port_failed_lock: + if result is False: + _port_failed_at[snode.get_id()] = (snode.cluster_id, time.monotonic()) + elif result is True: + _port_failed_at.pop(snode.get_id(), None) + + +def _correlated_port_failure(cluster, snode) -> str: + """Multi-attach clusters (only): why a failed port check of ``snode`` must + not set it DOWN, or "". Nodes that pass every liveness probe and fail only + the port check, more of them at once than the cluster tolerates, point at a + device or data-path condition (one bad LUN fences every LVS of a 1+0 + cluster), not at the nodes: setting them DOWN breaks the node floor and + suspends the cluster (2026-11-20 FTT2 bed). Same rule as the LUN + watchdog's observer guard.""" + if not multi_attach.is_multi_attach(cluster): + return "" + now = time.monotonic() + cid = cluster.get_id() + with _port_failed_lock: + failing = sorted(nid for nid, (c, t) in _port_failed_at.items() + if c == cid and now - t <= PORT_FAILURE_CORRELATION_SEC) + ftt = max(1, int(getattr(cluster, "max_fault_tolerance", 1) or 1)) + if len(failing) <= ftt: + return "" + reason = (f"{len(failing)} nodes fail only their port check at once ({failing}), more " + f"than the cluster tolerates ({ftt}): a data-path condition, not node failures") + if now - _correlated_event_at.get(cid, 0) > 60: + _correlated_event_at[cid] = now + try: + storage_events.snode_port_check_correlated(snode, reason) + except Exception as e: # noqa: BLE001 + logger.warning("Correlated port-check event not sent: %s", e) + return reason + + def node_port_check_fun(snode): node_port_check = True if snode.lvstore_status == "ready": @@ -2263,11 +2467,19 @@ def node_port_check_fun(snode): port_lvs_owner[_p] = snode.get_id() own_port = _p + # Multi-attach (1+0): a port SPDK reports blocked is its own fence + # (failed IO queued, leadership change), and one bad device fences + # every LVS of the cluster at once. The node's own port gets the + # grace peer-owned ports have, so the leak remediation can act before + # the block counts against the node (2026-11-20 FTT2 bed: three + # healthy nodes set DOWN within 4 s of such a fence). + own_port_grace = _multi_attach_cluster(snode.cluster_id) + # Batched: one nvmf_get_blocked_ports fetch answers every port, so # carrying the advisory ports costs no extra RPC. try: port_results = health_controller.check_ports_on_node(snode, ports) - if own_port is not None and port_results.get(own_port) is False: + if own_port is not None and port_results.get(own_port) is False and not own_port_grace: _reread_blocked_own_port(snode, own_port, port_results) for port, ret in port_results.items(): if port in advisory_ports: @@ -2276,13 +2488,14 @@ def node_port_check_fun(snode): f"(advisory: peer owns this LVS and may be fencing it)") continue owner_id = port_lvs_owner.get(port) - if ret is False and owner_id and owner_id != snode.get_id(): + if ret is False and owner_id and (owner_id != snode.get_id() or own_port_grace): held = _blocked_port_age(snode, port) if held < PEER_PORT_FENCE_GRACE_SEC: + whose = "peer-owned" if owner_id != snode.get_id() else "own" logger.info( f"Check: node port {snode.mgmt_ip}, {port} ... {ret} " - f"(peer-owned LVS fenced {held:.0f}s, within the " - f"{PEER_PORT_FENCE_GRACE_SEC:.0f}s takeover grace; not counted)") + f"({whose} LVS fenced {held:.0f}s, within the " + f"{PEER_PORT_FENCE_GRACE_SEC:.0f}s fence grace; not counted)") continue logger.info(f"Check: node port {snode.mgmt_ip}, {port} ... {ret}") node_port_check &= ret @@ -2752,6 +2965,8 @@ def check_node(snode): snode = db.get_storage_node_by_id(snode.get_id()) if snode.status == StorageNode.STATUS_ONLINE: storage_network.publish_ports(snode, at_most_every=PORT_PUBLISH_INTERVAL_SEC) + _reintegrate_fenced_luns(snode) + _ensure_warm_standbys(snode) # Self-heal a node stranded in IN_SHUTDOWN. A completed shutdown flips the # node to OFFLINE; if it is still IN_SHUTDOWN well past the grace window the @@ -3012,9 +3227,14 @@ def check_node(snode): else: node_port_check = port_check_holder["result"] + _note_port_check(snode, node_port_check) if node_port_check is False: cluster = db.get_cluster_by_id(snode.cluster_id) if cluster.status in [Cluster.STATUS_ACTIVE, Cluster.STATUS_DEGRADED, Cluster.STATUS_READONLY]: + held = _correlated_port_failure(cluster, snode) + if held: + logger.error("Port check failed on %s, node not set down: %s", snode.get_id(), held) + return True logger.error("Port check failed") set_node_down(snode) return True @@ -3041,6 +3261,90 @@ def check_node(snode): ) storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_ONLINE) readmit_devices_after_node_online(snode.get_id()) + _retire_taken_over_luns(snode) + + +def _retire_taken_over_luns(snode): + """Multi-attach clusters (only): a node back ONLINE without a restart + drops what it still holds of LUNs taken over while it was unreachable + (lun_takeover.retire_stale_local_stack).""" + try: + if not multi_attach.is_multi_attach(db.get_cluster_by_id(snode.cluster_id)): + return + from simplyblock_core.controllers import lun_takeover + lun_takeover.retire_stale_local_stack(snode) + except Exception as e: # noqa: BLE001 + logger.error("Retiring taken-over LUN stacks on %s failed: %s", snode.get_id(), e) + + +_reintegrate_last: dict = {} +_reintegrate_threads: dict = {} +REINTEGRATE_INTERVAL_SEC = 30 + + +def _reintegrate_fenced_luns(snode): + """Multi-attach clusters (only): an ONLINE node a takeover fenced from a + LUN becomes a standby of it again (lun_takeover.reintegrate_fenced_node), + checked at most every REINTEGRATE_INTERVAL_SEC per node.""" + node_id = snode.get_id() + if not any(multi_attach.fenced_from(d, node_id) + for n in db.get_storage_nodes_by_cluster_id(snode.cluster_id) + for d in n.nvme_devices or []): + return + now = time.monotonic() + if now - _reintegrate_last.get(node_id, 0) < REINTEGRATE_INTERVAL_SEC: + return + t = _reintegrate_threads.get(node_id) + if t is not None and t.is_alive(): + return + _reintegrate_last[node_id] = now + + def _run(): + try: + from simplyblock_core.controllers import lun_takeover + lun_takeover.reintegrate_fenced_node(snode) + except Exception as e: # noqa: BLE001 + logger.error("Reintegrating fenced LUNs of %s failed: %s", node_id, e) + # Off the node check: its agent calls on a node that is just lost held + # the liveness checks 30 s (2026-11-20 FTT2 bed, hard reset of sn3). + t = threading.Thread(target=_run, name=f"reintegrate-{node_id[:8]}", daemon=True) + _reintegrate_threads[node_id] = t + t.start() + + +_warm_standby_last: dict = {} +_warm_standby_threads: dict = {} + + +def _ensure_warm_standbys(snode): + """Warm-standby multi-attach clusters (only, Cluster.multi_attach_warm_standby): + an ONLINE node keeps a standby alceml of every LUN it is an attached + standby of, and closes the ones it is no longer a standby of + (lun_takeover.ensure_warm_standbys), at most every + MULTI_ATTACH_WARM_STANDBY_INTERVAL_SEC per node, in its own thread.""" + try: + cluster = db.get_cluster_by_id(snode.cluster_id) + except Exception as e: # noqa: BLE001 + logger.debug("Warm standbys of %s: cluster not read: %s", snode.get_id(), e) + return + if not (multi_attach.is_multi_attach(cluster) + and getattr(cluster, "multi_attach_warm_standby", False) is True): + return + node_id = snode.get_id() + now = time.monotonic() + if now - _warm_standby_last.get(node_id, 0) < constants.MULTI_ATTACH_WARM_STANDBY_INTERVAL_SEC: + return + t = _warm_standby_threads.get(node_id) + if t is not None and t.is_alive(): + return + _warm_standby_last[node_id] = now + + def _run(): + from simplyblock_core.controllers import lun_takeover + lun_takeover.ensure_warm_standbys(snode) + t = threading.Thread(target=_run, name=f"warm-standby-{node_id[:8]}", daemon=True) + _warm_standby_threads[node_id] = t + t.start() def readmit_devices_after_node_online(node_id): @@ -3173,6 +3477,9 @@ def main(): logger.info("Starting node monitor") threads_maps: dict[str, threading.Thread] = {} _start_housekeeping() + # Multi-attach clusters only; idle otherwise. + from simplyblock_core.services import lun_watchdog + lun_watchdog.start(_start_lun_takeover) while True: clusters = db.get_clusters() diff --git a/simplyblock_core/services/tasks_runner_failed_migration.py b/simplyblock_core/services/tasks_runner_failed_migration.py index 6c850959c9..ea376bef85 100644 --- a/simplyblock_core/services/tasks_runner_failed_migration.py +++ b/simplyblock_core/services/tasks_runner_failed_migration.py @@ -45,7 +45,8 @@ def task_runner(task): task.function_params["distr_name"], device.cluster_device_order, mig.qos_high_priority(snode.cluster_id), job_size=constants.MIG_JOB_SIZE, jobs=constants.MIG_PARALLEL_JOBS, - relaxed_mode=mig.migration_relaxed(snode.cluster_id))) + relaxed_mode=mig.migration_relaxed(snode.cluster_id), + affinity_floor=mig.migration_affinity_floor(snode.cluster_id))) if started is None: raise TaskAbort("canceled while starting migration") task = started diff --git a/simplyblock_core/services/tasks_runner_migration.py b/simplyblock_core/services/tasks_runner_migration.py index c4bfe0d2b7..3aa01c7cb9 100644 --- a/simplyblock_core/services/tasks_runner_migration.py +++ b/simplyblock_core/services/tasks_runner_migration.py @@ -109,6 +109,7 @@ def task_runner(task): task.function_params["distr_name"], mig.qos_high_priority(snode.cluster_id), job_size=constants.MIG_JOB_SIZE, jobs=constants.MIG_PARALLEL_JOBS, relaxed_mode=mig.migration_relaxed(snode.cluster_id), + affinity_floor=mig.migration_affinity_floor(snode.cluster_id), keep_placed=bool(getattr(db.get_cluster_by_id(snode.cluster_id), "page_copy", False))), migration_devices=_online_device_count(task.cluster_id)) if started is None: diff --git a/simplyblock_core/services/tasks_runner_new_dev_migration.py b/simplyblock_core/services/tasks_runner_new_dev_migration.py index 046193f3da..728771c8ba 100644 --- a/simplyblock_core/services/tasks_runner_new_dev_migration.py +++ b/simplyblock_core/services/tasks_runner_new_dev_migration.py @@ -68,7 +68,8 @@ def task_runner(task): started = mig.start_migration(task, lambda: rpc_client.distr_migration_expansion_start( task.function_params["distr_name"], mig.qos_high_priority(snode.cluster_id), job_size=constants.MIG_JOB_SIZE, jobs=constants.MIG_PARALLEL_JOBS, - relaxed_mode=mig.migration_relaxed(snode.cluster_id))) + relaxed_mode=mig.migration_relaxed(snode.cluster_id), + affinity_floor=mig.migration_affinity_floor(snode.cluster_id))) if started is None: raise TaskAbort("canceled while starting migration") task = started diff --git a/simplyblock_core/snode_client.py b/simplyblock_core/snode_client.py index 29ed7fa149..be0adc6ff0 100644 --- a/simplyblock_core/snode_client.py +++ b/simplyblock_core/snode_client.py @@ -257,6 +257,17 @@ def wipe_block_device(self, device_name): return self._request("POST", "wipe_block_device", {"device_name": device_name}) + def lun_info(self, wwn): + """Multi-attach: the local disks carrying LUN ``wwn``, each with + ``has_alceml_signature`` (block 0 read, nothing written). + Returns ``(result, error)``.""" + return self._request("POST", "lun_info", {"wwn": wwn}) + + def scan_new_luns(self): + """Have the host look for LUNs newly presented by a storage array. + Returns ``(result, error)``.""" + return self._request("POST", "scan_new_luns") + def rescan_block_device(self, device_path): """Re-read the size of a grown whole-disk block device (lblk). diff --git a/simplyblock_core/storage_arrays/README.md b/simplyblock_core/storage_arrays/README.md new file mode 100644 index 0000000000..10aa886214 --- /dev/null +++ b/simplyblock_core/storage_arrays/README.md @@ -0,0 +1,88 @@ +# External storage arrays + +Provision, resize and delete volumes (LUNs and NVMe namespaces) on enterprise arrays, and map them to hosts. +There is no usable cross-vendor standard, so each array has its own adapter. SMI-S is deprecated, Swordfish has almost no array support, and CSI drivers depend on Kubernetes. + +## Supported systems + +| Key | Systems | API | Protocols | +|---|---|---|---| +| `pure-flasharray` | Pure FlashArray //X, //C, //XL, //E | REST 2.x, API token | iSCSI, FC, NVMe/TCP, /RoCE, /FC | +| `netapp-ontap` | NetApp AFF/FAS/ASA (not ASA r2), Lenovo DM | ONTAP REST | iSCSI, FC, NVMe/TCP, /FC (`protocol_family`) | +| `dell-powerstore` | Dell PowerStore | REST | iSCSI, FC, NVMe/TCP, /FC | +| `dell-powermax` | Dell PowerMax | Unisphere REST | iSCSI, FC, NVMe (port-group per protocol) | +| `dell-powerflex` | Dell PowerFlex 3.6 / 4.x | Gateway REST / PFxM OAuth | SDC | +| `hpe-wsapi` | HPE Alletra Storage MP B10000, Primera, 3PAR | WSAPI | iSCSI, FC | +| `hpe-nimble` | HPE Alletra 5000/6000, Nimble | NimbleOS REST | iSCSI, FC | +| `ibm-flashsystem` | IBM FlashSystem, Storage Virtualize/SVC | REST (port 7443) | iSCSI, FC, NVMe/TCP, /RDMA | +| `hitachi-vsp` | Hitachi VSP One Block, VSP E, VSP 5000 | Configuration Manager REST | iSCSI, FC | +| `huawei-dorado` | Huawei OceanStor Dorado | DeviceManager REST | iSCSI, FC | +| `infinidat-infinibox` | Infinidat InfiniBox | REST | iSCSI, FC | + +Each adapter's module docstring lists the options it reads, the credential fields it needs, and a section called **"Verify against a real system"**. +That section names every API detail its author was unsure of. Work through it before the adapter is first used against a real array. + +## Using an array with a cluster + +The adapters are the array plugins of the multi-attach array LUN mode (`simplyblock_core.array.san`, one plugin per key above). +Register the array on a multi-attach cluster: + +``` +sbctl cluster set-array --plugin pure-flasharray --endpoint https://pure01.example \ + --options '{"protocol": "iscsi", "ca_bundle": "-----BEGIN CERTIFICATE-----..."}' \ + --credentials-file pure01.json --lun-size 4T +``` + +- `protocol` (required): `iscsi`, `fc`, `nvme-tcp`, `nvme-rdma`, `nvme-fc` or `sdc`. The nodes are registered on the array with the initiators of that protocol, as their agents report them. +- `ca_bundle`, `verify_tls`, `timeout_s`, plus the adapter's own options (pool, SVM, CPG, port group, ...; see the adapter docstring). +- Credentials: `--credentials-file` (wrapped by the cluster's KMS, bound to the array id and endpoint; the local KMS only with `allow_local_kms_credentials`), or `--credential-ref` naming a Vault KV / KMIP secret simplyblock only reads. + +Each node added afterwards gets a LUN of `--lun-size` created, mapped to its host, and found after a SCSI/NVMe rescan; the LUN is presented to the node's secondary and tertiary as well. +The hosts must already have their SAN paths: iSCSI sessions logged in, FC zoning done, NVMe-oF controllers connected. +LUNs are named `sb-`; a cluster never acts on a LUN of another name. + +## Layers + +- `base.py`: the adapter contract (`ArrayAdapter`) and the shared model. +- `client.py`: `ArrayClient`, the only entry point for the rest of simplyblock. It handles: + - credentials through `simplyblock_core.credentials`, bound to the array id and endpoint, with re-authentication and one retry on auth failure; + - `sb-` volume names and rounding sizes to the array's granularity; + - making create, map, unmap and delete safe to re-run; + - safety checks before resize, map or delete: the `sb-` name prefix, the name derived from the recorded volume UUID, and the recorded WWN/NGUID; + - refusing to shrink a volume. +- `http.py`: TLS (pinned CA bundle per array), timeouts, retries, error mapping. +- `registry.py`: maps vendor keys and aliases to adapters, loading each adapter on first use. + +## Testing + +1. **Contract suite against fakes** (`tests/unit/storage_arrays/test_contract.py`). + `testing/.py` is a stateful in-memory fake of the vendor's REST API: paths, JSON shapes, status codes, error bodies, async jobs, session tokens. + Every adapter runs the same suite against its fake, through a `requests` transport, with no network. + This shows the adapter's logic is right. It does **not** show the fake matches the real API; the fakes encode our reading of each vendor's API. +2. **Vendor unit tests** (`test_.py`): unit conversions, identifier derivation, job polling and failures, error mapping, options. +3. **HTTPS mock server**, for end-to-end and UI work: + ``` + python -m simplyblock_core.storage_arrays.testing --vendor netapp-ontap --port 8443 + ``` + It prints the endpoint, the CA file, the credential and the options to register. +4. **Live tier**: the same contract suite against a simulator or a real array: + ``` + SB_ARRAY_LIVE_CONFIG=lab-array.json python -m pytest tests/unit/storage_arrays/test_contract.py --timeout-method=thread + ``` + See `tests/unit/storage_arrays/conftest.py` for the file format. The live tier only creates and deletes `sb-` volumes and `sbtest-` hosts. + +### Where to get something real to test against + +These are partly from memory; check availability with each vendor. + +- **NetApp:** ONTAP Select evaluation, and the ONTAP simulator (support-site login). Both have the full REST API. A Lab on Demand is also an option. +- **Pure:** Pure Test Drive labs. There is no downloadable FlashArray simulator. +- **Dell:** PowerFlex can be installed on 3 VMs. Dell demo labs cover PowerStore and PowerMax. PowerStore has a virtual edition for partners. +- **HPE:** the 3PAR StoreServ simulator (older WSAPI). Alletra through HPE demo and partner labs. +- **IBM:** Storage Virtualize for Public Cloud, plus IBM Technology Zone labs. +- **Hitachi, Huawei:** partner and demo labs only. +- **Infinidat:** partner and demo labs. (`infinisim`, the simulator infinisdk's own tests use, is not published.) + +Vendor SDKs make good reference code for API details: `py-pure-client`, `netapp-ontap`, `PyPowerStore`, `PyU4V`, `PyPowerFlex`, `hpe3parclient`, `nimble-sdk`, `infinisdk`. +The same goes for each vendor's Ansible collection, CSI driver and OpenStack Cinder driver. +This package uses plain `requests` and does not depend on any of them. diff --git a/simplyblock_core/storage_arrays/__init__.py b/simplyblock_core/storage_arrays/__init__.py new file mode 100644 index 0000000000..d0ada602c8 --- /dev/null +++ b/simplyblock_core/storage_arrays/__init__.py @@ -0,0 +1,44 @@ +"""External storage arrays: vendor adapters behind one contract. + +Entry point: :class:`simplyblock_core.storage_arrays.client.ArrayClient`. +""" +from .base import ( + ArrayAdapter, + ArrayAuthError, + ArrayConfig, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + Capabilities, + HostSpec, + Initiator, + Mapping, + Protocol, + Volume, +) +from .registry import ADAPTERS, ALIASES, adapter_class, canonical_vendor + +__all__ = [ + "ADAPTERS", + "ALIASES", + "ArrayAdapter", + "ArrayAuthError", + "ArrayConfig", + "ArrayConflictError", + "ArrayError", + "ArrayInfo", + "ArrayNotFoundError", + "ArrayRequestError", + "ArrayUnsupportedError", + "Capabilities", + "HostSpec", + "Initiator", + "Mapping", + "Protocol", + "Volume", + "adapter_class", + "canonical_vendor", +] diff --git a/simplyblock_core/storage_arrays/adapters/__init__.py b/simplyblock_core/storage_arrays/adapters/__init__.py new file mode 100644 index 0000000000..9763b23894 --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/__init__.py @@ -0,0 +1 @@ +"""Vendor adapters; see ..registry for the vendor keys.""" diff --git a/simplyblock_core/storage_arrays/adapters/dell_powerflex.py b/simplyblock_core/storage_arrays/adapters/dell_powerflex.py new file mode 100644 index 0000000000..93bbf4806c --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/dell_powerflex.py @@ -0,0 +1,404 @@ +"""Dell PowerFlex (formerly ScaleIO / VxFlex OS), REST gateway API, PowerFlex +3.6 and 4.x. + +Auth (option ``auth``): + +* ``token`` (3.x gateway): ``GET /api/login`` with HTTP basic + ``username:password`` returns a session token as a JSON *string*; every + later request uses basic auth ``username:``. The token expires after + 8 h, or 10 min idle (-> 401; the client logs in again). + ``GET /api/logout`` invalidates it. +* ``oauth`` (4.x, PowerFlex Manager gateway): ``POST /rest/auth/login`` + ``{"username", "password"}`` returns ``{"access_token", "refresh_token", + ...}``; requests carry ``Authorization: Bearer ``. The + ``/api/...`` paths below are unchanged behind the 4.x gateway. + ``POST /rest/auth/logout {"refresh_token"}`` ends the session. The access + token is short-lived (minutes); rather than refreshing it, the adapter + logs in again on the 401 (``ArrayClient`` does this transparently). +* ``auto`` (default): ``GET /api/login`` as for ``token`` (served by 3.x + and 4.x gateways alike), then ``GET /api/version`` (a JSON string such as + ``"3.6"`` / ``"4.5"``): below 4.0 the token goes in basic auth as for + ``token``, from 4.0 on as ``Authorization: Bearer ``. This is how + Dell's own clients pick the scheme (goscaleio ``Authenticate`` / + ``BearerAuthenticationMinVersion``; PyPowerFlex ``get_api_version``). + +Volumes are addressed by their 16-hex-digit id (``Volume::``). Sizes +are in KiB on create and GiB on resize and are rounded up by the array to a +multiple of 8 GiB, so the adapter declares 8 GiB granularity and minimum. +Names are at most 31 characters. Volumes are created thin +(``ThinProvisioned``) in the ``storage_pool_id`` option's pool. + +Hosts: PowerFlex's native protocol is the SDC (storage data client) kernel +driver, which registers itself with the MDM when installed on the host. The +adapter never creates SDCs: ``ensure_host`` looks the SDC up by its GUID +(the initiator address, ``drv_cfg --query_guid`` on the host) and returns +its id as the host id. An SDC is one OS instance, so a host has exactly one +SDC GUID; specs with more are refused (:class:`ArrayUnsupportedError`). + +SDC names stand in for host ownership (option ``sdc_naming``): + +* ``claim`` (default): an unnamed SDC is named after the host + (``setSdcName``); an SDC already carrying the *same* name is that host; + an SDC carrying a *different* name belongs to another host and raises + :class:`ArrayConflictError` (an admin-named SDC must therefore be + registered under its PowerFlex name). +* ``ignore``: names are neither read nor written; any host name maps to the + SDC with the given GUID (no ownership conflict detection). + +``delete_host`` is a no-op for SDCs: the SDC belongs to the host's PowerFlex +installation (removing it is ``removeSdc``, an admin decision) and keeps the +name ``claim`` gave it. + +Mappings are SDC mappings (``addMappedSdc`` / ``removeMappedSdc``); the +volume's ``mappedSdcInfo`` lists them (``null``, not ``[]``, when there are +none). Mapping a volume to a second SDC needs ``allowMultipleMappings`` +(sent as ``"true"``, as csi-powerflex does). +SDC mappings have no LUN number (``Mapping.lun`` is ``None``) and SDC +volumes no NAA WWN or NGUID, so ``Volume.identifier`` is ``id.``. + +NVMe/TCP (4.x NVMe hosts, ``/api/types/Host/instances``) is not +implemented: a 4.x NVMe host has exactly one NQN and is mapped with +``addMappedHost`` rather than ``addMappedSdc`` (see below). + +Errors: the gateway answers ``{"message", "httpStatusCode", "errorCode"}``, +and reports "not found" as HTTP **500** (sometimes 400) rather than 404, so +missing objects are recognised by message text; any other 500 stays an +:class:`ArrayRequestError`. Booleans and numbers in action bodies are sent as +strings (``"true"``, ``"8388608"``), as Dell's clients send them. + +Checked against vendor source +----------------------------- +(PyPowerFlex = github.com/dell/python-powerflex, goscaleio = +github.com/dell/goscaleio, csi-powerflex = github.com/dell/csi-powerflex, +ansible-powerflex = github.com/dell/ansible-powerflex) + +* ``GET /api/login`` (basic user:password -> JSON string token), basic + ``user:`` on 3.x, ``GET /api/logout`` (PyPowerFlex + ``base_client.py`` ``_login`` / ``_logout``); a 4.x gateway still serves + ``/api/login`` and takes that token as a Bearer token (goscaleio + ``api.go`` ``basicAuthenticate``, ``api/api.go``); ``GET /api/version`` + picks the scheme (PyPowerFlex ``is_version_3``). +* 4.x OAuth: ``POST /rest/auth/login {"username", "password"}`` -> + ``{"access_token", "refresh_token"}``, ``POST /rest/auth/logout + {"refresh_token"}`` answering 204 (PyPowerFlex ``_appliance_login`` / + ``_appliance_logout``). +* Error texts: "Could not find the volume" (csi-powerflex + ``sioGatewayVolumeNotFound``); ``queryIdByKey`` for a missing name answers + "Not found" (goscaleio ``instance.go`` ``GetVolume``); "Volume name + already in use. Please use a different name." + (``sioGatewayVolumeNameInUse``). +* Gen1 arrays create volumes in multiples of 8 GiB, rounding up + (csi-powerflex ``validateVolSize``); Gen2/EC arrays use 1 GiB, which the + 8 GiB granularity also satisfies. +* ``addMappedSdc {"sdcId", "allowMultipleMappings", "accessMode"}`` with + ``accessMode`` ``ReadWrite`` (goscaleio ``MapVolumeSdcParam``; PyPowerFlex + ``add_mapped_sdc``; ansible-powerflex ``get_access_mode``); + ``allowMultipleMappings`` as a boolean string (csi-powerflex + ``strconv.FormatBool``, PyPowerFlex ``str(bool)``). +* ``removeMappedSdc``, ``setVolumeSize {"sizeInGB"}``, ``removeVolume + {"removeMode": "ONLY_ME"}``, ``queryIdByKey {"name"}``, create body + ``volumeSizeInKb`` / ``storagePoolId`` / ``volumeType`` as strings + (goscaleio ``sdc.go``, ``volume.go``, ``instance.go``, + ``types/v1/types.go`` ``VolumeParam``). +* Fields ``Sdc.sdcGuid`` / ``Sdc.name``, ``Volume.mappedSdcInfo[].sdcId``, + ``System.systemVersionName`` (goscaleio ``types/v1/types.go``). +* NVMe hosts map with ``addMappedHost {"hostId", ...}`` (goscaleio + ``nvme_host.go`` ``MapVolumeNVMe``) -- not implemented here. + +Verify against a real system +---------------------------- +* ``auth=oauth`` against a 3.x gateway (assumed: 404 for + ``/rest/auth/login``); whether ``GET /api/logout`` ends a Bearer session + on 4.x (the adapter calls it best-effort). +* 4.x token lifetime and whether a refresh-token grant would be preferable + to re-login. +* Status codes (500 vs 400 vs 404) and the wording for "already mapped", + "not mapped" and an SDC name in use. +* That ``setVolumeSize`` rounds a non-multiple of 8 up rather than + rejecting it (the adapter always sends multiples of 8, so this only + matters for volumes created outside simplyblock). +* Whether SDC names are unique system-wide (the fake assumes so). +""" +from __future__ import annotations + +import re +from typing import Any + +from ..base import ( + ArrayAdapter, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, +) +from ..http import body_text, json_of + +GIB = 1 << 30 +KIB = 1 << 10 + +_AUTH_MODES = ("auto", "token", "oauth") +_NAMING_MODES = ("claim", "ignore") +_NOT_FOUND = ("could not find", "not found", "does not exist", "no such") +_CONFLICT = ("already in use", "already exists", "already mapped", "is mapped", + "name in use") +_NOT_MAPPED = ("not mapped",) + + +def _json_string(text: str | None) -> str: + """A body that is a bare JSON string (``"abc"``), unquoted.""" + return (text or "").strip().strip('"') + + +def _message(body: Any, fallback: str) -> str: + if isinstance(body, dict) and body.get("message"): + msg = str(body["message"]) + code = body.get("errorCode") + return f"{msg} (errorCode {code})" if code is not None else msg + return fallback + + +class DellPowerFlexAdapter(ArrayAdapter): + vendor = "dell-powerflex" + display_name = "Dell PowerFlex" + capabilities = Capabilities( + protocols=frozenset({Protocol.SDC}), + size_granularity_bytes=8 * GIB, + min_size_bytes=8 * GIB, + max_name_len=31, + max_initiators_per_host=1, + ) + option_docs = ( + OptionDoc("storage_pool_id", "Id of the storage pool new volumes are created in.", + required=True), + OptionDoc("protection_domain_id", "Protection domain of the storage pool " + "(sent on create; only needed where the gateway asks for it)."), + OptionDoc("auth", "'token' (3.x /api/login, basic auth), 'oauth' (4.x " + "/rest/auth/login) or 'auto' (/api/login, then basic or Bearer by " + "/api/version).", default="auto"), + OptionDoc("sdc_naming", "'claim': name unnamed SDCs after the host and treat an " + "SDC named differently as another host's; 'ignore': never read or " + "write SDC names.", default="claim"), + ) + + def __init__(self, ctx): + super().__init__(ctx) + if self.option("auth") not in _AUTH_MODES: + raise ArrayError(f"PowerFlex option auth must be one of {', '.join(_AUTH_MODES)}") + if self.option("sdc_naming") not in _NAMING_MODES: + raise ArrayError( + f"PowerFlex option sdc_naming must be one of {', '.join(_NAMING_MODES)}") + self._mode = "" + self._refresh_token = "" + + # -- session ------------------------------------------------------------ + + def _reset_auth(self) -> None: + self.session.clear_auth() + self.session.headers.pop("Authorization", None) + self._mode = "" + self._refresh_token = "" + + def login(self) -> None: + self._reset_auth() + mode = self.option("auth") + user, password = self.secret("username"), self.secret("password") + if mode == "oauth": + resp = self.session.post("/rest/auth/login", + json={"username": user, "password": password}) + body = json_of(resp) or {} + token = body.get("access_token") if isinstance(body, dict) else None + if not token: + raise ArrayRequestError("PowerFlex /rest/auth/login returned no access_token") + self.session.headers["Authorization"] = f"Bearer {token}" + self._refresh_token = str(body.get("refresh_token") or "") + self._mode = "oauth" + return + resp = self.session.get("/api/login", auth=(user, password)) + token = _json_string(resp.text) + if not token: + raise ArrayRequestError("PowerFlex /api/login returned no token") + if mode == "auto": + version = _json_string(self.session.get("/api/version", auth=(user, token)).text) + m = re.match(r"^(\d+)\.", version) + if not m: + raise ArrayRequestError(f"PowerFlex /api/version returned {version!r}") + if int(m.group(1)) >= 4: + # goscaleio: 4.x takes the /api/login token as a Bearer token. + self.session.headers["Authorization"] = f"Bearer {token}" + self._mode = "bearer" + return + self.session.set_basic_auth(user, token) + self._mode = "token" + + def close(self) -> None: + try: + if self._mode in ("token", "bearer"): + self.session.get("/api/logout") + elif self._mode == "oauth": + self.session.post("/rest/auth/logout", + json={"refresh_token": self._refresh_token}) + except Exception: + pass + self._mode = "" + + # -- request helpers ----------------------------------------------------- + + def _req(self, method: str, path: str, *, json: Any = None, + missing_ok: bool = False) -> Any: + """Issue a request; returns the decoded body, or ``None`` for a + missing object when ``missing_ok``.""" + resp = self.session.request(method, path, json=json, ok=(400, 404, 500)) + if resp.status_code < 300: + if not resp.content: + return None + try: + return resp.json() + except ValueError: + return resp.text + try: + body = resp.json() + except ValueError: + body = None + msg = _message(body, body_text(resp)) + lowered = msg.lower() + if resp.status_code == 404 or any(s in lowered for s in _NOT_FOUND): + if missing_ok: + return None + raise ArrayNotFoundError(f"{method} {path}: {msg}") + if any(s in lowered for s in _CONFLICT): + raise ArrayConflictError(f"{method} {path}: {msg}") + raise ArrayRequestError(f"{method} {path} -> HTTP {resp.status_code}: {msg}", + status=resp.status_code) + + def _raw_volume(self, volume_id: str, missing_ok: bool = False) -> dict[str, Any] | None: + body = self._req("GET", f"/api/instances/Volume::{volume_id}", missing_ok=missing_ok) + if body is None: + return None + if not isinstance(body, dict): + raise ArrayRequestError(f"PowerFlex returned no volume for {volume_id}") + return body + + @staticmethod + def _volume(v: dict[str, Any]) -> Volume: + return Volume(id=str(v["id"]), name=v.get("name") or "", + size_bytes=int(v["sizeInKb"]) * KIB) + + def _action(self, volume_id: str, action: str, body: dict[str, Any]) -> Any: + return self._req("POST", f"/api/instances/Volume::{volume_id}/action/{action}", + json=body) + + # -- contract ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + systems = self._req("GET", "/api/types/System/instances") or [] + system = systems[0] if isinstance(systems, list) and systems else {} + return ArrayInfo(vendor=self.vendor, model="PowerFlex", + version=str(system.get("systemVersionName") or ""), + serial=str(system.get("id") or "")) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + body: dict[str, Any] = { + "name": name, + "volumeSizeInKb": str(size_bytes // KIB), + "storagePoolId": self.option("storage_pool_id"), + "volumeType": "ThinProvisioned", + } + if self.option("protection_domain_id"): + body["protectionDomainId"] = self.option("protection_domain_id") + created = self._req("POST", "/api/types/Volume/instances", json=body) + vid = created.get("id") if isinstance(created, dict) else None + if not vid: + raise ArrayRequestError("PowerFlex volume create returned no id") + raw = self._raw_volume(vid) + assert raw is not None + return self._volume(raw) + + def get_volume(self, volume_id: str) -> Volume | None: + raw = self._raw_volume(volume_id, missing_ok=True) + return self._volume(raw) if raw else None + + def find_volume(self, name: str) -> Volume | None: + vid = self._req("POST", "/api/types/Volume/instances/action/queryIdByKey", + json={"name": name}, missing_ok=True) + if not vid or not isinstance(vid, str): + return None + return self.get_volume(vid) + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + size_gb = -(-size_bytes // GIB) + self._action(volume_id, "setVolumeSize", {"sizeInGB": str(size_gb)}) + raw = self._raw_volume(volume_id) + assert raw is not None + return self._volume(raw) + + def delete_volume(self, volume_id: str) -> None: + self._action(volume_id, "removeVolume", {"removeMode": "ONLY_ME"}) + + # -- hosts (SDCs) ----------------------------------------------------------- + + def _sdcs(self) -> list[dict[str, Any]]: + body = self._req("GET", "/api/types/Sdc/instances") + return list(body) if isinstance(body, list) else [] + + def ensure_host(self, spec: HostSpec) -> str: + guids = spec.addresses(Protocol.SDC) + if len(guids) != 1 or len(spec.initiators) != 1: + raise ArrayUnsupportedError( + "A PowerFlex host is one SDC: give exactly one SDC GUID per host") + guid = guids[0] + sdc = next((s for s in self._sdcs() + if str(s.get("sdcGuid") or "").lower() == guid.lower()), None) + if sdc is None: + raise ArrayNotFoundError( + f"No SDC with GUID {guid} is registered with PowerFlex; install the SDC " + "on the host and connect it to the MDM first") + sdc_id = str(sdc["id"]) + if self.option("sdc_naming") == "claim": + current = sdc.get("name") or "" + if current and current != spec.name: + raise ArrayConflictError( + f"SDC {guid} is registered as host {current!r}, not {spec.name!r}") + if not current: + self._req("POST", f"/api/instances/Sdc::{sdc_id}/action/setSdcName", + json={"sdcName": spec.name}) + return sdc_id + + def delete_host(self, host_id: str) -> None: + """No-op: SDCs belong to the host's PowerFlex installation.""" + + # -- mappings --------------------------------------------------------------- + + def list_mappings(self, volume_id: str) -> list[Mapping]: + raw = self._raw_volume(volume_id) + assert raw is not None + info = raw.get("mappedSdcInfo") or [] # null when unmapped + return [Mapping(volume_id=volume_id, host_id=str(m["sdcId"]), lun=None) + for m in info if m.get("sdcId")] + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + for m in self.list_mappings(volume_id): + if m.host_id == host_id: + return m + self._action(volume_id, "addMappedSdc", { + "sdcId": host_id, "allowMultipleMappings": "true", "accessMode": "ReadWrite"}) + return Mapping(volume_id=volume_id, host_id=host_id, lun=None) + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + raw = self._raw_volume(volume_id, missing_ok=True) + if raw is None: + return + if not any(str(m.get("sdcId")) == host_id for m in raw.get("mappedSdcInfo") or []): + return + try: + self._action(volume_id, "removeMappedSdc", {"sdcId": host_id}) + except ArrayRequestError as e: + if not any(s in str(e).lower() for s in _NOT_MAPPED): + raise diff --git a/simplyblock_core/storage_arrays/adapters/dell_powermax.py b/simplyblock_core/storage_arrays/adapters/dell_powermax.py new file mode 100644 index 0000000000..19de6dfce0 --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/dell_powermax.py @@ -0,0 +1,495 @@ +"""Dell PowerMax, Unisphere for PowerMax REST API (10.x). + +Auth: HTTP basic on every request (no session). Provisioning lives under +``/univmax/restapi/{api_version}/sloprovisioning/symmetrix/{symmetrix_id}``. +Errors are ``{"message": "..."}``, mostly with status 400; 404 for missing +objects, 409 for "already exists", some 500s; 401 for a rejected credential. +They are classified by wording (see below). + +Layout (one object set per simplyblock volume): + +* storage group ```` (``sb-``) in the configured SRP / + service level, holding exactly one device; +* the device's ``volume_identifier`` is the same name; +* one masking view per (storage group, host): ``_`` (or + ``_`` when that exceeds 64 characters), joining the storage + group, the host (initiator group) and the port group configured for the + host's protocol. + +A storage group per volume keeps map/unmap/delete independent per volume +(PowerMax maps storage groups, not devices), at the cost of one SG and one +masking view per mapping; arrays allow tens of thousands of each. + +``Volume.id`` is the 5-hex-digit device id (``0012A``). ``Volume.wwn`` is +``naa.`` + the device's ``effective_wwn`` (falls back to ``wwn``). Volume +identifiers are *not* unique on PowerMax; lookups by name prefer the device +in the storage group of that name and refuse to guess between several. + +Sizes: devices are allocated in cylinders; on PowerMaxOS (FBA) one cylinder +is 15 tracks x 128 KiB = 1920 KiB, so the size granularity is 1 966 080 +bytes and all sizes are sent as ``capacityUnit: "CYL"`` (exact, no GB/MB +rounding). ``cap_cyl`` is read back. + +Async jobs: with ``async_jobs`` the adapter sends +``executionOption: ASYNCHRONOUS`` and polls ``/{ver}/system/job/{jobId}`` on +``202 Accepted``; otherwise it sends ``SYNCHRONOUS`` but still follows a 202 +should the server answer with a job. + +Checked against vendor source +----------------------------- +(PyU4V = github.com/dell/PyU4V, gopowermax = github.com/dell/gopowermax, +csi-powermax = github.com/dell/csi-powermax) + +* ``sloId: "None"`` for no service level, with ``workloadSelection: "None"`` + (PyU4V ``provisioning.py`` ``create_storage_group``; gopowermax + ``GetCreateStorageGroupPayload``). The adapter sends both. +* An empty ``GET .../volume?volume_identifier=x`` answers ``count: 0`` + without ``resultList`` (PyU4V ``get_volume_list`` reads ``resultList`` + only when ``count > 0``); the filter is an exact match, ```` is the + substring form (gopowermax ``GetVolumeIDsIterator``); ``storageGroupId`` + filter on the same endpoint (``GetVolumesInStorageGroupIterator``). +* Error body ``{"message"}`` (gopowermax ``types.Error``). Missing objects + read "Could not find volume ...", "... cannot be found" (csi-powermax + ``controller.go`` ``cannotBeFound`` / ``errorNotFound``), removing a + device its storage group lacks "does not contain" (``deletion_worker.go``); + all three are classified as missing. "already exists" is a conflict. +* Delete order: remove the device from its storage group, then + ``DELETE volume`` -- no deallocation step (csi-powermax deletion worker + states ``disAssociateSG`` -> ``deleteVolume``; PyU4V ``delete_volume``). +* iSCSI and NVMe hosts are created before the node logs in to the array + (csi-powermax ``setupArrayForIscsi`` / ``setupArrayForNVMeTCP``). +* NVMe host initiator ids are ``:`` with the host ID as + 32 upper-case hex digits, not the bare NQN (csi-powermax + ``makeNVMeInitiatorIDs``). The adapter accepts an address already in that + form, or derives the host ID from a ``nqn.2014-08.org.nvmexpress:uuid:`` + NQN; other NQNs are refused (:class:`ArrayUnsupportedError`). +* Volume fields ``nguid``, ``effective_wwn``, ``storageGroupId``, + ``cap_cyl`` (gopowermax ``types/v100/volume.go``); 1 CYL = 1 966 080 + bytes (csi-powermax ``cylinderSizeInBytes``). +* ``maskingview/{mv}/connections?volume_id=`` rows carry + ``host_lun_address`` in hex (csi-powermax ``node.go`` + ``strconv.ParseInt(..., 16, 0)``); a view may have no rows yet + (csi-powermax vSphere path), giving ``lun=None``. +* Jobs: ``SUCCEEDED`` is done; ``CREATED``/``SCHEDULED``/``RUNNING``/ + ``VALIDATING``/``VALIDATED`` are in progress; any other state is a failure + whose text is in ``result`` (PyU4V ``common.py`` ``_is_job_finished``, + ``constants.INCOMPLETE_LIST``). + +Verify against a real system +---------------------------- +* Status codes: whether "already exists" comes as 409 or 400 and "cannot be + found" on masking view create as 404 or 500 (both handled), and the + wording for a shrink, removing the last device of a masked SG, deleting a + device still in an SG and an initiator in another host. +* Whether FC WWNs must be visible to the array (logged in / zoned) before + ``POST host`` accepts them; csi-powermax only registers WWNs the array + already lists in ``GET .../initiator``. +* That ``/etc/nvme/hostid`` equals the UUID in a uuid-form host NQN (the + nvme-cli default) on the hosts in use; otherwise pass ``:``. +* Whether ``volume_identifier`` matching is case-sensitive; iterator paging + (``/common/Iterator``) is not followed -- fine for lookups by name or + storage group that match one device. +""" +from __future__ import annotations + +import hashlib +import math +import re +from collections.abc import Callable +from typing import Any + +import requests + +from ..base import ( + ArrayAdapter, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, +) +from ..http import json_of + +CYLINDER_BYTES = 1920 * 1024 +MAX_NAME = 64 +_JOB_DONE = "SUCCEEDED" +# PyU4V constants.INCOMPLETE_LIST; any other non-SUCCEEDED state is terminal. +_JOB_RUNNING = frozenset({"CREATED", "SCHEDULED", "RUNNING", "VALIDATING", "VALIDATED"}) +_JOB_POLL_S = 2.0 + +_NOT_FOUND = ("cannot find", "could not find", "cannot be found", "not found", + "does not exist", "does not contain", "no such") +_CONFLICT = ("already", "in use", "exists", "member of", "is part of", "masking view") + + +def _message(resp: requests.Response) -> str: + try: + body = json_of(resp) + except ArrayRequestError: + body = None + if isinstance(body, dict) and body.get("message"): + return str(body["message"]) + return (resp.text or "")[:600] or f"HTTP {resp.status_code}" + + +def _classify(what: str, text: str, status: int | None = None) -> ArrayError: + lowered = text.lower() + msg = f"{what}: {text}" + if status == 404 or any(k in lowered for k in _NOT_FOUND): + return ArrayNotFoundError(msg) + if any(k in lowered for k in _CONFLICT): + return ArrayConflictError(msg) + return ArrayRequestError(msg, status=status) + + +def powermax_wwn(item: dict[str, Any]) -> str | None: + v = str(item.get("effective_wwn") or item.get("wwn") or "").strip().lower() + v = v.removeprefix("naa.") + return f"naa.{v}" if v else None + + +_HOST_ID_RE = re.compile(r"^(?Pnqn\..+):(?P[0-9A-Fa-f]{32})$") +_UUID_NQN_RE = re.compile( + r"^nqn\.2014-08\.org\.nvmexpress:uuid:(?P[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-" + r"[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12})$") + + +def powermax_nvme_initiator(address: str) -> str: + """PowerMax NVMe initiator id: ``:`` (csi-powermax + ``makeNVMeInitiatorIDs``), HOSTID being the 32 upper-case hex digits of + ``/etc/nvme/hostid``. Accepts that form, or a uuid-form NQN whose UUID + is taken as the host ID.""" + m = _HOST_ID_RE.match(address) + if m: + return f"{m.group('nqn')}:{m.group('hostid').upper()}" + m = _UUID_NQN_RE.match(address) + if m: + return f"{address}:{m.group('uuid').replace('-', '').upper()}" + raise ArrayUnsupportedError( + f"PowerMax needs the NVMe host ID with the host NQN: give {address!r} as " + "':<32-hex host ID from /etc/nvme/hostid>'") + + +def initiator_protocol(initiator_id: str) -> Protocol: + """Protocol of an initiator-group member as Unisphere lists it.""" + v = initiator_id.lower() + if v.startswith(("iqn.", "eui.")): + return Protocol.ISCSI + if v.startswith("nqn."): + return Protocol.NVME_TCP + return Protocol.FC + + +class DellPowerMaxAdapter(ArrayAdapter): + vendor = "dell-powermax" + display_name = "Dell PowerMax" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC, Protocol.NVME_TCP}), + size_granularity_bytes=CYLINDER_BYTES, + min_size_bytes=CYLINDER_BYTES, + max_name_len=MAX_NAME, + ) + credential_fields = (frozenset({"username", "password"}),) + option_docs = ( + OptionDoc("symmetrix_id", "12-digit array serial (symmetrix id), e.g. '000197900123'.", + required=True), + OptionDoc("srp", "Storage resource pool for new storage groups.", default="SRP_1"), + OptionDoc("service_level", "Service level (e.g. 'Diamond'); default: none."), + OptionDoc("port_group", "Port group for masking views: a name, or a map from " + "protocol ('iscsi', 'fc', 'nvme-tcp') to name. Needed to map volumes."), + OptionDoc("api_version", "Unisphere REST version in the URL ('100' = 10.0, " + "'101' = 10.1, ...).", default="100"), + OptionDoc("async_jobs", "Run modifying calls as Unisphere jobs and poll them " + "(recommended for large arrays where synchronous calls time out).", + default=False), + OptionDoc("job_timeout_s", "How long to wait for an asynchronous job.", default=600), + ) + + def __init__(self, ctx): + super().__init__(ctx) + self._base = f"/univmax/restapi/{self.option('api_version')}" + self._sym = str(self.option("symmetrix_id")) + self._slo = f"{self._base}/sloprovisioning/symmetrix/{self._sym}" + self._sleep: Callable[[float], None] = self.session.sleep + + # -- session ------------------------------------------------------------ + + def login(self) -> None: + self.session.set_basic_auth(self.secret("username"), self.secret("password")) + + def close(self) -> None: + self.session.clear_auth() + + # -- request helpers ----------------------------------------------------- + + def _exec(self) -> str: + return "ASYNCHRONOUS" if self.option("async_jobs") else "SYNCHRONOUS" + + def _call(self, method: str, path: str, *, params: dict[str, Any] | None = None, + json: Any = None, missing_ok: bool = False) -> Any: + # 409: "already exists"; 500: e.g. "Port Group on Symmetrix cannot be + # found" on masking view create. Both are classified by wording. + resp = self.session.request(method, path, params=params, json=json, + ok=(400, 404, 409, 500)) + if resp.status_code in (400, 404, 409, 500): + err = _classify(f"{method} {path} -> HTTP {resp.status_code}", _message(resp), + resp.status_code) + if missing_ok and isinstance(err, ArrayNotFoundError): + return None + raise err + body = json_of(resp) + if resp.status_code == 202 and isinstance(body, dict) and body.get("jobId"): + return self._wait_job(str(body["jobId"]), f"{method} {path}") + return body + + def _wait_job(self, job_id: str, what: str) -> dict[str, Any]: + polls = max(1, math.ceil(float(self.option("job_timeout_s")) / _JOB_POLL_S)) + for _ in range(polls): + job = json_of(self.session.get(f"{self._base}/system/job/{job_id}")) or {} + status = str(job.get("status", "")).upper() + if status == _JOB_DONE: + return job + if status not in _JOB_RUNNING: + raise _classify(f"{what} (job {job_id} {status or 'no status'})", + str(job.get("result") or "no result")) + self._sleep(_JOB_POLL_S) + raise ArrayRequestError(f"{what}: job {job_id} did not finish within " + f"{self.option('job_timeout_s')} s") + + @staticmethod + def _volume(item: dict[str, Any]) -> Volume: + nguid = str(item.get("nguid") or "").strip().lower().removeprefix("eui.") + return Volume(id=item["volumeId"], name=item.get("volume_identifier") or "", + size_bytes=int(item["cap_cyl"]) * CYLINDER_BYTES, + wwn=powermax_wwn(item), nguid=f"eui.{nguid}" if nguid else None) + + def _volume_item(self, device_id: str) -> dict[str, Any] | None: + return self._call("GET", f"{self._slo}/volume/{device_id}", missing_ok=True) + + def _require_item(self, device_id: str) -> dict[str, Any]: + item = self._volume_item(device_id) + if item is None: + raise ArrayNotFoundError(f"PowerMax device {device_id} not found") + return item + + def _device_ids(self, **filters: str) -> list[str]: + body = self._call("GET", f"{self._slo}/volume", params=filters) or {} + rows = (body.get("resultList") or {}).get("result") or [] + return [r["volumeId"] for r in rows if r.get("volumeId")] + + @staticmethod + def _cyl(size_bytes: int) -> int: + return -(-int(size_bytes) // CYLINDER_BYTES) + + def _attrs(self, name: str, size_bytes: int) -> dict[str, Any]: + return {"num_of_vols": 1, "volume_size": str(self._cyl(size_bytes)), + "capacityUnit": "CYL", + "volumeIdentifier": {"identifier_name": name, + "volumeIdentifierChoice": "identifier_name"}} + + # -- contract ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + about = self._call("GET", "/univmax/restapi/version") or {} + supported = about.get("supported_api_versions") + if supported and str(self.option("api_version")) not in [str(v) for v in supported]: + raise ArrayUnsupportedError( + f"Unisphere {about.get('version', '')} does not serve REST version " + f"{self.option('api_version')} (supported: {', '.join(map(str, supported))})") + sym = self._call("GET", f"{self._base}/system/symmetrix/{self._sym}") or {} + version = sym.get("ucode") or "" + unisphere = about.get("version") or "" + return ArrayInfo(vendor=self.vendor, model=sym.get("model", ""), + version=f"PowerMaxOS {version} (Unisphere {unisphere})".strip(), + serial=sym.get("symmetrixId", self._sym)) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + attrs = self._attrs(name, size_bytes) + payload: dict[str, Any] = { + "srpId": self.option("srp"), + "storageGroupId": name, + "emulation": "FBA", + "sloBasedStorageGroupParam": [{ + "sloId": self.option("service_level") or "None", + "workloadSelection": "None", + "volumeAttributes": [attrs], + }], + "executionOption": self._exec(), + } + try: + self._call("POST", f"{self._slo}/storagegroup", json=payload) + except ArrayConflictError: + # Left over from an interrupted create or delete: reuse it if empty. + sg = self._call("GET", f"{self._slo}/storagegroup/{name}", missing_ok=True) + if sg is None or int(sg.get("num_of_vols") or 0): + raise + self._call("PUT", f"{self._slo}/storagegroup/{name}", json={ + "editStorageGroupActionParam": {"expandStorageGroupParam": {"addVolumeParam": { + "create_new_volumes": True, "emulation": "FBA", + "volumeAttributes": [attrs]}}}, + "executionOption": self._exec()}) + devices = self._device_ids(storageGroupId=name) + if len(devices) != 1: + raise ArrayRequestError( + f"PowerMax storage group {name} holds {len(devices)} devices after create") + return self._volume(self._require_item(devices[0])) + + def get_volume(self, volume_id: str) -> Volume | None: + item = self._volume_item(volume_id) + return self._volume(item) if item else None + + def find_volume(self, name: str) -> Volume | None: + items = [i for i in (self._volume_item(d) for d in self._device_ids(volume_identifier=name)) + if i and i.get("volume_identifier") == name] + if len(items) > 1: + own = [i for i in items if name in (i.get("storageGroupId") or [])] + if len(own) != 1: + raise ArrayConflictError( + f"{len(items)} PowerMax devices are named {name}: " + + ", ".join(i["volumeId"] for i in items)) + items = own + return self._volume(items[0]) if items else None + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + self._call("PUT", f"{self._slo}/volume/{volume_id}", json={ + "editVolumeActionParam": {"expandVolumeParam": {"volumeAttribute": { + "volume_size": str(self._cyl(size_bytes)), "capacityUnit": "CYL"}}}, + "executionOption": self._exec()}) + return self._volume(self._require_item(volume_id)) + + def delete_volume(self, volume_id: str) -> None: + item = self._require_item(volume_id) + name = item.get("volume_identifier") or "" + for sg in item.get("storageGroupId") or []: + self._call("PUT", f"{self._slo}/storagegroup/{sg}", json={ + "editStorageGroupActionParam": {"removeVolumeParam": {"volumeId": [volume_id]}}, + "executionOption": self._exec()}, missing_ok=True) + if sg == name: + self._call("DELETE", f"{self._slo}/storagegroup/{sg}", missing_ok=True) + self._call("DELETE", f"{self._slo}/volume/{volume_id}") + + # -- hosts ------------------------------------------------------------------ + + def ensure_host(self, spec: HostSpec) -> str: + wanted = [powermax_nvme_initiator(i.address) if i.protocol.is_nvme else i.normalized() + for i in spec.initiators] + host = self._call("GET", f"{self._slo}/host/{spec.name}", missing_ok=True) + if host is None: + self._call("POST", f"{self._slo}/host", json={ + "hostId": spec.name, "initiatorId": wanted, "executionOption": self._exec()}) + return spec.name + have = {str(i).lower() for i in host.get("initiator") or []} + missing = [w for w in wanted if w.lower() not in have] + if missing: + self._call("PUT", f"{self._slo}/host/{spec.name}", json={ + "editHostActionParam": {"addInitiatorParam": {"initiator": missing}}, + "executionOption": self._exec()}) + return spec.name + + def delete_host(self, host_id: str) -> None: + self._call("DELETE", f"{self._slo}/host/{host_id}", missing_ok=True) + + # -- mappings ------------------------------------------------------------------ + + @staticmethod + def view_name(sg: str, host: str) -> str: + name = f"{sg}_{host}" + if len(name) <= MAX_NAME: + return name + return f"{sg}_{hashlib.sha256(host.encode()).hexdigest()[:16]}" + + def _port_group(self, host_id: str) -> str: + option = self.option("port_group") + if not option: + raise ArrayError(f"{self.display_name} array {self.config.id} needs the " + "'port_group' option to map volumes") + if isinstance(option, str): + return option + host = self._call("GET", f"{self._slo}/host/{host_id}") + protocols = {initiator_protocol(str(i)) for i in host.get("initiator") or []} + if len(protocols) != 1: + raise ArrayError(f"Cannot tell the protocol of PowerMax host {host_id}") + protocol = protocols.pop() + pg = option.get(protocol.value) + if not pg: + raise ArrayError(f"No PowerMax port group configured for {protocol.value} " + f"(option 'port_group')") + return str(pg) + + def _views(self, item: dict[str, Any]) -> list[tuple[str, str, str]]: + """(masking view, host or host group, storage group) for a device.""" + out = [] + for sg in item.get("storageGroupId") or []: + sg_item = self._call("GET", f"{self._slo}/storagegroup/{sg}", missing_ok=True) + for mv in (sg_item or {}).get("maskingview") or []: + mv_item = self._call("GET", f"{self._slo}/maskingview/{mv}", missing_ok=True) + if mv_item: + host = mv_item.get("hostId") or mv_item.get("hostGroupId") + if host: + out.append((mv, str(host), sg)) + return out + + def _lun(self, view: str, device_id: str) -> int | None: + body = self._call("GET", f"{self._slo}/maskingview/{view}/connections", + params={"volume_id": device_id}, missing_ok=True) or {} + for row in body.get("maskingViewConnection") or []: + if row.get("volumeId") == device_id and row.get("host_lun_address"): + return int(str(row["host_lun_address"]), 16) + return None + + def _mappings(self, item: dict[str, Any]) -> list[Mapping]: + seen: dict[str, Mapping] = {} + for view, host, _sg in self._views(item): + if host not in seen: + seen[host] = Mapping(volume_id=item["volumeId"], host_id=host, + lun=self._lun(view, item["volumeId"])) + return list(seen.values()) + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + item = self._require_item(volume_id) + for m in self._mappings(item): + if m.host_id == host_id: + return m + sg = item.get("volume_identifier") or "" + if sg not in (item.get("storageGroupId") or []): + raise ArrayConflictError( + f"PowerMax device {volume_id} is not in its own storage group {sg!r}") + if self._call("GET", f"{self._slo}/host/{host_id}", missing_ok=True) is None: + raise ArrayNotFoundError(f"PowerMax host {host_id} not found") + self._call("POST", f"{self._slo}/maskingview", json={ + "maskingViewId": self.view_name(sg, host_id), + "hostOrHostGroupSelection": {"useExistingHostParam": {"hostId": host_id}}, + "portGroupSelection": {"useExistingPortGroupParam": { + "portGroupId": self._port_group(host_id)}}, + "storageGroupSelection": {"useExistingStorageGroupParam": {"storageGroupId": sg}}, + "executionOption": self._exec()}) + for m in self._mappings(self._require_item(volume_id)): + if m.host_id == host_id: + return m + raise ArrayRequestError( + f"PowerMax masked device {volume_id} to host {host_id} but lists no mapping") + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + item = self._volume_item(volume_id) + if item is None: + return + own = item.get("volume_identifier") or "" + for view, host, sg in self._views(item): + if host != host_id: + continue + if sg != own: + raise ArrayConflictError( + f"PowerMax device {volume_id} is mapped to {host_id} through shared " + f"storage group {sg} (masking view {view}); unmap it on the array") + self._call("DELETE", f"{self._slo}/maskingview/{view}", missing_ok=True) + + def list_mappings(self, volume_id: str) -> list[Mapping]: + item = self._volume_item(volume_id) + return self._mappings(item) if item else [] diff --git a/simplyblock_core/storage_arrays/adapters/dell_powerstore.py b/simplyblock_core/storage_arrays/adapters/dell_powerstore.py new file mode 100644 index 0000000000..c85e3eacf7 --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/dell_powerstore.py @@ -0,0 +1,377 @@ +"""Dell PowerStore, REST API (PowerStoreOS 3.x / 4.x). + +Auth: HTTP basic on ``GET /api/rest/login_session`` opens a session. The +response carries the session cookie (``auth_cookie``) and a CSRF token in the +``DELL-EMC-TOKEN`` header; every later request sends the cookie, and every +non-GET request must also send the token header (the array answers 403 +without it). Basic auth is *not* kept on the session afterwards, so an expired +session surfaces as 401 or 403 and :class:`ArrayClient` logs in again. + +Queries use the PostgREST-style syntax: ``select=a,b`` picks fields (with +embedded objects as ``host(host_initiators)``), ``name=eq.x`` filters, +``limit``/``offset`` page; a partial page is answered with +``206 Partial Content`` and a ``Content-Range`` header. + +Errors: ``{"messages": [{"code": "0x...", "severity": "Error", +"message_l10n": "...", "arguments": [...]}]}``. Semantic failures (initiator +owned by another host, volume still attached, shrink) come back as +``422 Unprocessable Entity``; a duplicate volume name as 422 or 500; malformed +requests, a missing host and a detach of a volume that is not attached +(``"Host is not attached to volume"``) as 400. Errors are classified by +wording: "not attached" and "not found" mean missing, "already" / "in use" / +"attached" / "mapped" mean conflict. + +Objects are addressed by id (UUID) for volumes and hosts. Volume identity: +``wwn`` is returned as ``naa.``, ``nguid`` as ``nguid.<32 hex>``; this +adapter reports them as ``naa.`` / ``eui.`` (lowercase). + +Mappings: ``host_volume_mapping.logical_unit_number`` is the SCSI LUN. For +NVMe hosts the namespace id is the volume's own ``nsid`` field ("NVMe +Namespace unique identifier in the NVME subsystem"), so a mapping to a host +whose initiators are all ``NVMe`` reports that ``nsid`` (falling back to +``logical_unit_number`` when the array returns none). + +Sizes are bytes; PowerStore requires a multiple of 8 KiB (8192) and at least +1 MiB. + +Checked against vendor source +----------------------------- +(PyPowerStore = github.com/dell/python-powerstore, gopowerstore = +github.com/dell/gopowerstore, csi-powerstore = github.com/dell/csi-powerstore) + +* Session: ``auth_cookie`` cookie and ``DELL-EMC-TOKEN`` header from + ``GET login_session``, both sent on later requests, logout is + ``POST /logout`` (PyPowerStore ``client.py`` ``AuthenticationManager``); + gopowerstore re-logs in on 403 (``api/api.go`` ``Query``). +* Create answers ``{"id"}`` (gopowerstore ``CreateResponse``), modify / + delete / attach / detach an empty body (``EmptyResponse``); attach and + detach are ``POST volume/{id}/attach|detach {"host_id"}`` (PyPowerStore + ``constants.py`` ``MAP_VOLUME_TO_HOST_URL``). +* Error body ``messages[].message_l10n`` (gopowerstore ``api/api.go`` + ``ErrorMsg``); status codes per gopowerstore ``gopowerstore_types.go``: + name in use 422 or 500, delete of an attached volume 422, detach of a + non-attached volume / missing host 400 or 404; wording "Host is not + attached to volume" and "already registered with another host" + (csi-powerstore ``controller/base.go``, ``node/node.go``). +* Host fields ``host_initiators`` (GET), ``initiators`` (create), + ``add_initiators`` (modify), each ``{"port_name", "port_type"}`` with + ``port_type`` ``iSCSI``/``FC``/``NVMe`` (gopowerstore ``host_types.go``); + NVMe/TCP and NVMe/FC hosts both register their NQN as ``NVMe``, FC WWPNs + colon-separated lowercase (csi-powerstore ``buildInitiatorsArray``, + ``formatWWPN``). +* Volume ``nsid`` field (gopowerstore ``volume_types.go``) and embedded + selects on ``host_volume_mapping`` (``HostVolumeMapping.Fields``). +* ``name=eq.`` unquoted (gopowerstore ``GetHostByName``, + ``GetVolumeByName``). +* Size: multiple of 8192, minimum 1 MB (gopowerstore ``VolumeCreate``, + csi-powerstore ``VolumeSizeMultiple`` / ``MinVolumeSizeBytes``). + +Verify against a real system +---------------------------- +* Status and wording for a shrink, deleting a host with mapped volumes, and + re-adding an initiator already on the same host. +* Whether ``logical_unit_number`` on an NVMe host's mapping equals the + volume's ``nsid`` (the adapter prefers ``nsid``), and which LUN the array + picks first for SCSI (fake: 1). +* Exact behaviour of ``name=eq.`` for names containing commas or parentheses + (simplyblock names are ``sb-``; host names are caller-chosen). +""" +from __future__ import annotations + +from http.cookies import CookieError, SimpleCookie +from typing import Any + +import requests + +from ..base import ( + ArrayAdapter, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, + colon_wwpn, + normalize_wwpn, +) +from ..http import json_of + +API = "/api/rest" +TOKEN_HEADER = "DELL-EMC-TOKEN" +VOLUME_FIELDS = "id,name,size,wwn,nguid" +HOST_FIELDS = "id,name,host_initiators" +MAPPING_FIELDS = ("id,host_id,host_group_id,volume_id,logical_unit_number," + "host(host_initiators),volume(nsid)") +_PAGE = 1000 + +# "not attached": detach of a volume that is not (or no longer) attached. +_NOT_FOUND = ("not found", "does not exist", "not exist", "no such", "not attached") +_CONFLICT = ("already", "in use", "exists", "attached", "mapped", "is used", "duplicate") + + +def _messages(resp: requests.Response) -> str: + try: + body = json_of(resp) or {} + except ArrayRequestError: + return "" + if not isinstance(body, dict): + return "" + return " ".join(str(m.get("message_l10n") or m.get("code") or "") + for m in body.get("messages", []) if isinstance(m, dict)) + + +def _classify(method: str, path: str, resp: requests.Response) -> ArrayError: + text = _messages(resp) or f"HTTP {resp.status_code}" + lowered = text.lower() + msg = f"{method} {path} -> HTTP {resp.status_code}: {text}" + if resp.status_code == 404 or any(k in lowered for k in _NOT_FOUND): + return ArrayNotFoundError(msg) + if any(k in lowered for k in _CONFLICT): + return ArrayConflictError(msg) + return ArrayRequestError(msg, status=resp.status_code) + + +def powerstore_wwn(value: str | None) -> str | None: + v = (value or "").strip().lower().removeprefix("naa.") + return f"naa.{v}" if v else None + + +def powerstore_nguid(value: str | None) -> str | None: + v = (value or "").strip().lower().removeprefix("nguid.").removeprefix("eui.") + return f"eui.{v}" if v else None + + +_PORT_TYPE = { + Protocol.ISCSI: "iSCSI", + Protocol.FC: "FC", + Protocol.NVME_TCP: "NVMe", + Protocol.NVME_FC: "NVMe", +} + + +def _port_key(port_type: str, port_name: str) -> str: + """Comparable form of an initiator as the array reports it.""" + if port_type == "FC": + try: + return normalize_wwpn(port_name) + except ValueError: + return port_name.lower() + if port_type == "iSCSI": + return port_name.lower() + return port_name + + +def _is_nvme_host(host: Any) -> bool: + """True for an embedded ``host`` whose initiators are all NVMe.""" + ports = host.get("host_initiators") if isinstance(host, dict) else None + if not ports: + return False + return all(p.get("port_type") == "NVMe" for p in ports) + + +class DellPowerStoreAdapter(ArrayAdapter): + vendor = "dell-powerstore" + display_name = "Dell PowerStore" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC, Protocol.NVME_TCP, Protocol.NVME_FC}), + size_granularity_bytes=8192, + min_size_bytes=1 << 20, + max_name_len=128, + ) + credential_fields = (frozenset({"username", "password"}),) + option_docs = ( + OptionDoc("os_type", "os_type of hosts simplyblock creates.", default="Linux"), + OptionDoc("appliance_id", "Create volumes on this appliance of a multi-appliance " + "cluster (e.g. 'A1'); default: the array places them."), + ) + + # -- session ------------------------------------------------------------ + + def login(self) -> None: + self.session.headers.pop(TOKEN_HEADER, None) + self.session.http.cookies.clear() + resp = self.session.get(f"{API}/login_session", + auth=(self.secret("username"), self.secret("password"))) + token = resp.headers.get(TOKEN_HEADER) + if not token: + raise ArrayRequestError(f"PowerStore login returned no {TOKEN_HEADER} header") + self.session.headers[TOKEN_HEADER] = token + self._adopt_cookies(resp) + + def _adopt_cookies(self, resp: requests.Response) -> None: + """Keep the session cookie. ``requests`` normally stores it already; + this also covers transports that do not populate the cookie jar.""" + raw = resp.headers.get("Set-Cookie") + if not raw: + return + jar = self.session.http.cookies + try: + parsed = SimpleCookie() + parsed.load(raw) + except CookieError: + return + for name, morsel in parsed.items(): + if name not in jar: + jar.set(name, morsel.value, path=morsel["path"] or "/") + + def close(self) -> None: + if TOKEN_HEADER in self.session.headers: + try: + self.session.post(f"{API}/logout", ok=(401, 403)) + except Exception: + pass + self.session.headers.pop(TOKEN_HEADER, None) + + # -- request helpers ----------------------------------------------------- + + def _call(self, method: str, path: str, *, params: dict[str, Any] | None = None, + json: Any = None, missing_ok: bool = False) -> requests.Response | None: + # 500 included: a duplicate volume name may come back as 500, not 422. + resp = self.session.request(method, f"{API}{path}", params=params, json=json, + ok=(400, 404, 422, 500)) + if resp.status_code in (400, 404, 422, 500): + err = _classify(method, path, resp) + if missing_ok and isinstance(err, ArrayNotFoundError): + return None + raise err + return resp + + def _one(self, path: str, select: str) -> dict[str, Any] | None: + resp = self._call("GET", path, params={"select": select}, missing_ok=True) + return json_of(resp) if resp is not None else None + + def _list(self, path: str, select: str, **filters: str) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + offset = 0 + while True: + params: dict[str, Any] = {"select": select, **{k: f"eq.{v}" for k, v in filters.items()}, + "limit": _PAGE, "offset": offset} + resp = self.session.request("GET", f"{API}{path}", params=params, + ok=(206, 400, 422)) + if resp.status_code in (400, 422): + raise _classify("GET", path, resp) + items = json_of(resp) or [] + out.extend(items) + if resp.status_code != 206 or not items: + return out + offset += len(items) + + @staticmethod + def _volume(item: dict[str, Any]) -> Volume: + return Volume(id=item["id"], name=item["name"], size_bytes=int(item["size"]), + wwn=powerstore_wwn(item.get("wwn")), + nguid=powerstore_nguid(item.get("nguid"))) + + def _require_volume(self, volume_id: str) -> Volume: + vol = self.get_volume(volume_id) + if vol is None: + raise ArrayNotFoundError(f"PowerStore volume {volume_id} not found") + return vol + + # -- contract ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + clusters = self._list("/cluster", "id,name,global_id") + cluster = clusters[0] if clusters else {} + model = version = "" + try: + appliances = self._list("/appliance", "id,name,model") + model = next((a.get("model", "") for a in appliances if a.get("model")), "") + software = self._list("/software_installed", "id,release_version,is_cluster", + is_cluster="true") + version = next((s.get("release_version", "") for s in software), "") + except (ArrayRequestError, ArrayNotFoundError): + pass + return ArrayInfo(vendor=self.vendor, model=model, + version=f"PowerStoreOS {version}".strip(), + serial=cluster.get("global_id") or cluster.get("id", "")) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + body: dict[str, Any] = {"name": name, "size": size_bytes} + if self.option("appliance_id"): + body["appliance_id"] = self.option("appliance_id") + resp = self._call("POST", "/volume", json=body) + created = json_of(resp) if resp is not None else None + if not created or "id" not in created: + raise ArrayRequestError(f"PowerStore volume create for {name} returned no id") + return self._require_volume(created["id"]) + + def get_volume(self, volume_id: str) -> Volume | None: + item = self._one(f"/volume/{volume_id}", VOLUME_FIELDS) + return self._volume(item) if item else None + + def find_volume(self, name: str) -> Volume | None: + items = self._list("/volume", VOLUME_FIELDS, name=name) + return self._volume(items[0]) if items else None + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + self._call("PATCH", f"/volume/{volume_id}", json={"size": size_bytes}) + return self._require_volume(volume_id) + + def delete_volume(self, volume_id: str) -> None: + self._call("DELETE", f"/volume/{volume_id}") + + def _host_by_name(self, name: str) -> dict[str, Any] | None: + items = self._list("/host", HOST_FIELDS, name=name) + return items[0] if items else None + + def ensure_host(self, spec: HostSpec) -> str: + wanted: list[dict[str, str]] = [] + for i in spec.initiators: + port_name = colon_wwpn(i.address) if i.protocol == Protocol.FC else i.normalized() + wanted.append({"port_name": port_name, "port_type": _PORT_TYPE[i.protocol]}) + host = self._host_by_name(spec.name) + if host is None: + resp = self._call("POST", "/host", json={ + "name": spec.name, "os_type": self.option("os_type"), "initiators": wanted}) + created = json_of(resp) if resp is not None else None + if not created or "id" not in created: + raise ArrayRequestError(f"PowerStore host create for {spec.name} returned no id") + return str(created["id"]) + have = {_port_key(p.get("port_type", ""), p.get("port_name", "")) + for p in host.get("host_initiators") or []} + missing = [w for w in wanted if _port_key(w["port_type"], w["port_name"]) not in have] + if missing: + self._call("PATCH", f"/host/{host['id']}", json={"add_initiators": missing}) + return str(host["id"]) + + def delete_host(self, host_id: str) -> None: + self._call("DELETE", f"/host/{host_id}", missing_ok=True) + + def _mappings(self, volume_id: str) -> list[Mapping]: + items = self._list("/host_volume_mapping", MAPPING_FIELDS, volume_id=volume_id) + out = [] + for i in items: + host = i.get("host_id") or i.get("host_group_id") + if host: + lun = i.get("logical_unit_number") + if _is_nvme_host(i.get("host")): + lun = (i.get("volume") or {}).get("nsid") or lun + out.append(Mapping(volume_id=i["volume_id"], host_id=host, + lun=int(lun) if lun is not None else None)) + return out + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + for m in self._mappings(volume_id): + if m.host_id == host_id: + return m + self._call("POST", f"/volume/{volume_id}/attach", json={"host_id": host_id}) + for m in self._mappings(volume_id): + if m.host_id == host_id: + return m + raise ArrayRequestError( + f"PowerStore attached volume {volume_id} to host {host_id} but lists no mapping") + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + if not any(m.host_id == host_id for m in self._mappings(volume_id)): + return + self._call("POST", f"/volume/{volume_id}/detach", json={"host_id": host_id}, + missing_ok=True) + + def list_mappings(self, volume_id: str) -> list[Mapping]: + return self._mappings(volume_id) diff --git a/simplyblock_core/storage_arrays/adapters/hitachi_vsp.py b/simplyblock_core/storage_arrays/adapters/hitachi_vsp.py new file mode 100644 index 0000000000..8d3c14c9eb --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/hitachi_vsp.py @@ -0,0 +1,641 @@ +"""Hitachi VSP (VSP One Block, VSP E series, VSP 5000, VSP G/F) through the +Configuration Manager REST API. + +Deployment modes (option ``storage_device_id``): + +* embedded (option absent): the API runs on the SVP / controller and objects + live under ``/ConfigurationManager/v1/objects/...``; +* REST API server (Ops Center API Configuration Manager): the storage system + must already be registered on the server; objects live under + ``/ConfigurationManager/v1/objects/storages/{storageDeviceId}/...``. + +Auth: ``POST .../sessions`` with HTTP basic auth (a storage-system user) +returns ``{"token", "sessionId"}``; later requests send ``Authorization: +Session ``; ``DELETE .../sessions/{sessionId}`` discards it. An expired +session answers 401, which the client turns into one re-login. + +Asynchronous jobs: create/modify/delete requests answer ``202`` with a job +object (``jobId``, ``status``, ``state``, ``affectedResources``, ``error``). +The adapter polls ``GET .../jobs/{jobId}`` until ``status == "Completed"``; +``state`` is then ``Succeeded`` or ``Failed`` (``error`` carries the usual +error body). Polling gives up after ``job_timeout_s``; the job may still +finish on the array, and every operation is safe to re-run. + +Object model: + +* volume = DP-VOL LDEV in pool ``pool_id``; ``Volume.id`` is the decimal LDEV + number, ``Volume.name`` the LDEV label (max 32 characters, not unique on the + array). A deleted LDEV still answers ``GET /ldevs/{id}`` with + ``emulationType == "NOT DEFINED"``, which reads as "missing". +* host = one host group (FC) / iSCSI target (iSCSI) of the same name on every + configured port of the matching type; the host id is that name. Initiators + are ``host-wwns`` / ``host-iscsis`` of each of those groups. +* mapping = one LU path per port (``POST /luns``); the adapter uses the same + LUN number on every port and reports the LUN of the first configured port. + +Volume identity: SCSI ``naa.`` (lowercase). + +Checked against vendor source (Hitachi's OpenStack Cinder driver +``cinder/volume/drivers/hitachi/hbsd_rest.py`` / ``hbsd_rest_api.py`` and the +Ansible collection ``hitachivantara.vspone_block``, ``plugins/module_utils/ +gateway/gateway_manager.py``, ``vsp_volume.py``, ``vsp_host_group_gateway.py``, +``vsp_rg_lock_gateway.py``, ``common/vsp_constants.py``, ``vsp_utils.py``): + +* Sessions: ``POST .../sessions`` with basic auth -> ``token``/``sessionId``, + ``Authorization: Session ``, ``DELETE .../sessions/{id}``; 401 means + re-login. Jobs: 202 + ``jobId``, done at ``status == "Completed"``, + ``state == "Succeeded"``, new object = last segment of + ``affectedResources[0]``; a failed job's ``error`` is the usual body. +* ``POST /ldevs {poolId, blockCapacity}`` without ``ldevId`` auto-assigns + the number (``startLdevId``/``endLdevId`` restrict it); the label is set + afterwards with ``PUT /ldevs/{id} {"label"}``, exactly as Cinder does. + Expand: ``POST .../actions/expand/invoke {"parameters": + {"additionalBlockCapacity"}}``. A deleted LDEV reads ``emulationType == + "NOT DEFINED"``. +* Lookup by label: there is no label filter; Cinder scans ``GET /ldevs? + ldevOption=dpVolume&headLdevId=&count=16384`` stepping the head by + ``count`` up to LDEV 65535 and ignoring KART40044-E (window past the + end). This adapter does the same (it used to stop after the first short + page, which missed LDEVs above the first 16k numbers). +* NAA: derived from the storage serial and the LDEV number (Ansible + ``NAIDCalculator``), so a reused LDEV number has the same WWN. +* ``GET /ldevs/{id}`` lists LU paths under ``ports`` (``portId``, + ``hostGroupNumber``, ``hostGroupName``, ``lun``); ``GET /host-wwns`` / + ``/host-iscsis`` take ``portId`` + ``hostGroupNumber``; ``POST + /host-groups {portId, hostGroupName, hostMode[, hostModeOptions]}``; + ``POST /luns {portId, hostGroupNumber, ldevId, lun}``; ``DELETE + /luns/{port},{gid},{lun}``. +* Resource-group lock: ``.../resource-group-service/actions/lock/invoke`` + with ``{"parameters": {"waitTime"}}`` lives under + ``/ConfigurationManager/v1/services`` (embedded) or + ``/ConfigurationManager/v1/{storageDeviceId}/services`` (REST server), + not under ``/objects`` (fixed here). Ansible POSTs it, Cinder PUTs it. +* Error codes (classified before the text fallback): KART30013-E and + ``EX_ENOOBJ`` = not found; KART40050-E..52-E, SSB 2E11/2205, 2E11/2207, + ``EX_EACCES`` = resource group locked; SSB 2E22/0001 (LDEV already + defined), B958/015A (LU path already defined), B958/0947 (another LDEV on + that LUN) = conflict. Storage failures carry SSB1/SSB2 in ``errorCode``. + +Verify against a real system: + +* Older microcode that may require ``ldevId`` on ``POST /ldevs`` (set + ``ldev_range`` there). +* A crash between LDEV create and label leaves an unlabeled LDEV the client + cannot find (a label failure is cleaned up here, a crash is not). +* DP-VOL capacity: the adapter rounds to 1 MiB and creates at least 64 MiB + (Cinder only creates whole GiB); the real floor per model. +* Whether every model returns ``naaId`` (Ansible computes it when absent; + this adapter then reports no WWN). +* FC refusing a WWN in two host groups of one port and iSCSI allowing an + initiator in several targets (the ownership scan does not rely on either). +* Which models need the resource-group lock (``use_lock``); Cinder retries + lock errors for up to two hours, this adapter reports a conflict. +* NVMe-oF (``/nvm-subsystems``) is not implemented: only FC and iSCSI. +""" +from __future__ import annotations + +import re +import time +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any + +from ..base import ( + ArrayAdapter, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, +) +from ..http import json_of + +API = "/ConfigurationManager/" +ROOT = "/ConfigurationManager/v1/objects" +BLOCK = 512 +NOT_DEFINED = "NOT DEFINED" +MAX_LUN = 2047 +#: ``count`` per ``GET /ldevs`` page (the API maximum). ``headLdevId`` steps +#: by this much, as in Hitachi's Cinder driver (``hbsd_rest.py``). +PAGE_SIZE = 16384 +#: Highest LDEV number scanned when no ``ldev_range`` is set (Cinder's +#: ``_MAX_LDEV_ID``). +MAX_LDEV_ID = 65535 + +_PORT_RE = re.compile(r"^CL[0-9A-Z]+-[A-Z0-9]+$") +_RANGE_RE = re.compile(r"^\s*(\d+)\s*-\s*(\d+)\s*$") +_PORT_TYPES = {"FIBRE": Protocol.FC, "ISCSI": Protocol.ISCSI} +#: HTTP statuses whose Hitachi error body the adapter classifies itself +#: (401/403 stay with RestSession -> ArrayAuthError). +_OWN_STATUSES = (400, 404, 405, 409, 412, 500, 502, 503, 504) + +# Structured codes, from Hitachi's Cinder driver (hbsd_rest_api.py, hbsd_rest.py). +#: messageId of "the specified object does not exist". +_MSGID_NOT_FOUND = frozenset({"KART30013-E"}) +#: messageIds of a resource-group lock held by someone else. +_MSGID_LOCKED = frozenset({"KART40050-E", "KART40051-E", "KART40052-E"}) +#: (SSB1, SSB2) pairs: LDEV already defined, LU path already defined, another +#: LDEV already mapped to that LUN, resource group locked. +_SSB_CONFLICT = frozenset({("2E22", "0001"), ("B958", "015A"), ("B958", "0947"), + ("2E11", "2205"), ("2E11", "2207")}) +#: errorCode.errorCode values. +_RC_NOT_FOUND = frozenset({"EX_ENOOBJ"}) +_RC_CONFLICT = frozenset({"EX_EACCES"}) +#: A ``GET /ldevs`` window beyond the array's last LDEV number. +_MSGID_BAD_RANGE = "KART40044-E" +_RC_BAD_RANGE = ("EX_ENLDEV", "EX_INVARG") + +_CONFLICT_TEXT = ("already", "in use", "is used", "registered", "duplicat", + "lu path is defined", "lu paths are defined", "locked") +_MISSING_TEXT = ("does not exist", "not found", "not defined", "not exist") + + +def _monotonic() -> float: + return time.monotonic() + + +def _classify(where: str, status: int | None, body: Any) -> ArrayError: + """Map a Hitachi error body (``errorSource``, ``message``, ``solution``, + ``messageId``, ``errorCode``) to an ArrayError.""" + if isinstance(body, dict): + message = str(body.get("message") or body.get("error") or "") + msg_id = str(body.get("messageId") or "") + codes = body.get("errorCode") + ssb = "" + if isinstance(codes, dict) and codes: + ssb = " (" + ", ".join(f"{k}={v}" for k, v in sorted(codes.items())) + ")" + text = f"{where}: {msg_id + ' ' if msg_id else ''}{message}{ssb}".strip() + else: + message = str(body or "") + text = f"{where}: {message}" + if status: + text += f" [HTTP {status}]" + lowered = message.lower() + if isinstance(body, dict): + raw = body.get("errorCode") + err: dict[str, Any] = raw if isinstance(raw, dict) else {} + ssb_pair = (str(err.get("SSB1", "")).upper(), str(err.get("SSB2", "")).upper()) + rc = str(err.get("errorCode", "")) + mid = str(body.get("messageId") or "") + if mid in _MSGID_NOT_FOUND or rc in _RC_NOT_FOUND: + return ArrayNotFoundError(text) + if mid in _MSGID_LOCKED or ssb_pair in _SSB_CONFLICT or rc in _RC_CONFLICT: + return ArrayConflictError(text) + if status == 404: + return ArrayNotFoundError(text) + if status in (409, 412): + return ArrayConflictError(text) + if status is None or status in (400, 500): + if any(t in lowered for t in _MISSING_TEXT): + return ArrayNotFoundError(text) + if any(t in lowered for t in _CONFLICT_TEXT): + return ArrayConflictError(text) + return ArrayRequestError(text, status=status) + + +def _tail(resource: str) -> str: + """``/ConfigurationManager/v1/objects/ldevs/123`` -> ``123``.""" + return resource.rstrip("/").rsplit("/", 1)[-1] + + +class HitachiVspAdapter(ArrayAdapter): + vendor = "hitachi-vsp" + display_name = "Hitachi VSP" + capabilities = Capabilities( + protocols=frozenset({Protocol.FC, Protocol.ISCSI}), + size_granularity_bytes=1 << 20, + min_size_bytes=64 << 20, + # LDEV labels and iSCSI target names: 32 characters. + max_name_len=32, + ) + option_docs = ( + OptionDoc("pool_id", "DP (HDP/HDT) pool number volumes are created in.", + required=True), + OptionDoc("ports", "Target ports hosts are presented on, e.g. " + "['CL1-A', 'CL2-A']. FC and iSCSI ports may be mixed; a host " + "uses the ports of its initiators' protocol.", required=True), + OptionDoc("storage_device_id", "storageDeviceId (e.g. '886000123456') when " + "the endpoint is an Ops Center API Configuration Manager server; " + "leave unset for the API embedded on the SVP/controller."), + OptionDoc("ldev_range", "Restrict new LDEV numbers (and volume lookups) " + "to 'start-end' (decimal LDEV ids)."), + OptionDoc("host_mode", "hostMode of new host groups.", default="LINUX/IRIX"), + OptionDoc("host_mode_options", "hostModeOptions numbers for new host groups.", + default=()), + OptionDoc("job_timeout_s", "Seconds to wait for an asynchronous job.", + default=300), + OptionDoc("use_lock", "Take the resource-group lock around configuration " + "changes (needed on some models/configurations).", default=False), + OptionDoc("lock_wait_s", "waitTime for the resource-group lock.", default=30), + ) + + def __init__(self, ctx): + super().__init__(ctx) + ports = self.option("ports") + if isinstance(ports, str): + ports = [p for p in ports.replace(",", " ").split() if p] + if not ports or not all(isinstance(p, str) and _PORT_RE.match(p.upper()) + for p in ports): + raise ArrayError(f"Hitachi VSP option 'ports' must list port ids like " + f"'CL1-A'; got {self.option('ports')!r}") + self._ports: list[str] = list(dict.fromkeys(p.upper() for p in ports)) + try: + self._pool = int(self.option("pool_id")) + except (TypeError, ValueError) as e: + raise ArrayError("Hitachi VSP option 'pool_id' must be a number") from e + self._range: tuple[int, int] | None = None + rng = self.option("ldev_range") + if rng: + m = _RANGE_RE.match(str(rng)) + if not m or int(m.group(1)) > int(m.group(2)): + raise ArrayError(f"Hitachi VSP option 'ldev_range' must be 'start-end'; " + f"got {rng!r}") + self._range = (int(m.group(1)), int(m.group(2))) + sdid = self.option("storage_device_id") + self._base = f"{ROOT}/storages/{sdid}" if sdid else ROOT + # Services are not under /objects: /ConfigurationManager/v1/services + # (embedded, Hitachi Ansible vsp_rg_lock_gateway.py) or + # /ConfigurationManager/v1/{storageDeviceId}/services (REST API + # server, Hitachi Cinder hbsd_rest_api.py ``service_url``). + self._services = (f"{API}v1/{sdid}/services" if sdid + else f"{API}v1/services") + self._session_id: Any = None + self._port_types: dict[str, Protocol | None] = {} + + # -- session ------------------------------------------------------------ + + def login(self) -> None: + self.session.headers.pop("Authorization", None) + self._session_id = None + resp = self.session.post(f"{self._base}/sessions", + auth=(self.secret("username"), self.secret("password"))) + body = json_of(resp) or {} + token = body.get("token") + if not token: + raise ArrayRequestError("Hitachi VSP session request returned no token") + self._session_id = body.get("sessionId") + self.session.headers["Authorization"] = f"Session {token}" + + def close(self) -> None: + if self._session_id is not None and "Authorization" in self.session.headers: + try: + self.session.delete(f"{self._base}/sessions/{self._session_id}") + except Exception: + pass + self.session.headers.pop("Authorization", None) + self._session_id = None + + # -- request helpers ----------------------------------------------------- + + def _req(self, method: str, path: str, *, params: Any = None, + json: Any = None) -> tuple[int, Any]: + full = path if path.startswith(API) else f"{self._base}{path}" + resp = self.session.request(method, full, params=params, json=json, + ok=_OWN_STATUSES) + try: + body = json_of(resp) + except ArrayRequestError: + body = resp.text + if resp.status_code >= 400: + raise _classify(f"{method} {path}", resp.status_code, body) + return resp.status_code, body + + def _get(self, path: str, **params: Any) -> Any: + return self._req("GET", path, params=params or None)[1] + + def _data(self, path: str, **params: Any) -> list[dict[str, Any]]: + return list((self._get(path, **params) or {}).get("data", [])) + + def _run(self, method: str, path: str, json: Any = None) -> list[str]: + """Issue a change request, wait for its job and return the job's + ``affectedResources``.""" + status, body = self._req(method, path, json=json) + if not isinstance(body, dict) or "jobId" not in body: + return [] + job = self._wait(body) + if job.get("state") != "Succeeded": + raise _classify(f"{method} {path} (job {job.get('jobId')} {job.get('state')})", + None, job.get("error") or {"message": "job did not succeed"}) + return [str(r) for r in job.get("affectedResources") or []] + + def _wait(self, job: dict[str, Any]) -> dict[str, Any]: + timeout = float(self.option("job_timeout_s")) + sleep = self.session.sleep + deadline = _monotonic() + timeout + interval, waited = 0.5, 0.0 + job_id = job["jobId"] + while job.get("status") != "Completed": + if waited >= timeout or _monotonic() > deadline: + raise ArrayRequestError( + f"Hitachi VSP job {job_id} not completed after {timeout:g}s " + f"(status {job.get('status')}, state {job.get('state')}); " + "it may still finish on the array") + sleep(interval) + waited += interval + interval = min(interval * 2, 5.0) + job = self._get(f"/jobs/{job_id}") or {} + return job + + @contextmanager + def _change(self) -> Iterator[None]: + """Resource-group lock around a configuration change (``use_lock``).""" + if not self.option("use_lock"): + yield + return + svc = f"{self._services}/resource-group-service/actions" + self._run("POST", f"{svc}/lock/invoke", + {"parameters": {"waitTime": int(self.option("lock_wait_s"))}}) + try: + yield + finally: + try: + self._run("POST", f"{svc}/unlock/invoke", {}) + except ArrayError: + pass + + # -- volumes -------------------------------------------------------------- + + @staticmethod + def _defined(ldev: dict[str, Any] | None) -> bool: + return bool(ldev) and (ldev or {}).get("emulationType", NOT_DEFINED) != NOT_DEFINED + + @staticmethod + def _volume(ldev: dict[str, Any]) -> Volume: + naa = str(ldev.get("naaId") or "").lower() + return Volume(id=str(ldev["ldevId"]), name=str(ldev.get("label") or ""), + size_bytes=int(ldev["blockCapacity"]) * BLOCK, + wwn=f"naa.{naa}" if naa else None) + + def _ldev(self, volume_id: str) -> dict[str, Any] | None: + if not str(volume_id).isdigit(): + return None + try: + ldev = self._get(f"/ldevs/{int(volume_id)}") + except ArrayNotFoundError: + return None + return ldev if self._defined(ldev) else None + + def _require(self, volume_id: str) -> dict[str, Any]: + ldev = self._ldev(volume_id) + if ldev is None: + raise ArrayNotFoundError(f"LDEV {volume_id} is not defined") + return ldev + + def get_info(self) -> ArrayInfo: + sdid = self.option("storage_device_id") + info = self._get(f"{ROOT}/storages/{sdid}" if sdid else f"{ROOT}/storages/instance") + info = info or {} + return ArrayInfo(vendor=self.vendor, model=str(info.get("model", "")), + version=str(info.get("dkcMicroVersion", "")), + serial=str(info.get("serialNumber", ""))) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + body: dict[str, Any] = {"poolId": self._pool, "blockCapacity": size_bytes // BLOCK} + if self._range: + body["startLdevId"], body["endLdevId"] = self._range + with self._change(): + affected = self._run("POST", "/ldevs", body) + ldev_ids = [_tail(r) for r in affected if "/ldevs/" in r] + if not ldev_ids: + raise ArrayRequestError(f"Hitachi VSP LDEV create for {name} reported no LDEV") + ldev_id = ldev_ids[0] + try: + self._run("PUT", f"/ldevs/{ldev_id}", {"label": name}) + except ArrayError: + # An unlabeled LDEV cannot be found again; do not leak it. + try: + self._run("DELETE", f"/ldevs/{ldev_id}") + except ArrayError: + pass + raise + return self._volume(self._require(ldev_id)) + + def get_volume(self, volume_id: str) -> Volume | None: + ldev = self._ldev(volume_id) + return self._volume(ldev) if ldev else None + + def find_volume(self, name: str) -> Volume | None: + # ``count`` is a window of LDEV numbers starting at ``headLdevId``, so + # step the head by ``count`` (not by the last id returned) as Hitachi's + # Cinder driver does; a window past the array's last LDEV number + # answers KART40044-E / EX_ENLDEV and ends the scan. ``poolId`` is not + # combined with ``headLdevId`` (no vendor client does); the pool is + # checked on each entry instead. + start, end = self._range if self._range else (0, MAX_LDEV_ID) + found: dict[int, dict[str, Any]] = {} + head = start + while head <= end: + count = min(PAGE_SIZE, end - head + 1) + try: + page = self._data("/ldevs", ldevOption="dpVolume", headLdevId=head, + count=count) + except ArrayError as e: + past_end = _MSGID_BAD_RANGE in str(e) or any( + rc in str(e) for rc in _RC_BAD_RANGE) + if past_end and head > start: # never on the first window + break + raise + for ldev in page: + lid = int(ldev["ldevId"]) + if (start <= lid <= end and ldev.get("label") == name + and self._defined(ldev) + and int(ldev.get("poolId", self._pool)) == self._pool): + found[lid] = ldev + head += count + if len(found) > 1: + raise ArrayConflictError( + f"Several LDEVs labeled {name!r}: " + ", ".join(str(i) for i in sorted(found))) + if not found: + return None + # List entries may omit fields; read the LDEV itself. + return self.get_volume(str(next(iter(found)))) + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + ldev = self._require(volume_id) + add = size_bytes // BLOCK - int(ldev["blockCapacity"]) + if add > 0: + with self._change(): + self._run("POST", f"/ldevs/{ldev['ldevId']}/actions/expand/invoke", + {"parameters": {"additionalBlockCapacity": add}}) + return self._volume(self._require(volume_id)) + + def delete_volume(self, volume_id: str) -> None: + ldev = self._require(volume_id) + with self._change(): + self._run("DELETE", f"/ldevs/{ldev['ldevId']}") + + # -- ports and host groups ------------------------------------------------------ + + def _port_protocol(self, port: str) -> Protocol | None: + if port not in self._port_types: + info = self._get(f"/ports/{port}") or {} + self._port_types[port] = _PORT_TYPES.get(str(info.get("portType", "")).upper()) + return self._port_types[port] + + def _ports_for(self, protocol: Protocol) -> list[str]: + return [p for p in self._ports if self._port_protocol(p) == protocol] + + def _groups(self, port: str) -> list[dict[str, Any]]: + return self._data("/host-groups", portId=port) + + @staticmethod + def _by_name(groups: list[dict[str, Any]], name: str) -> dict[str, Any] | None: + return next((g for g in groups if g.get("hostGroupName") == name), None) + + def _host_groups(self, name: str) -> list[tuple[str, int]]: + """(port, hostGroupNumber) of host ``name`` on every configured port.""" + out = [] + for port in self._ports: + g = self._by_name(self._groups(port), name) + if g is not None: + out.append((port, int(g["hostGroupNumber"]))) + return out + + def _members(self, protocol: Protocol, port: str, number: int) -> set[str]: + if protocol == Protocol.FC: + items = self._data("/host-wwns", portId=port, hostGroupNumber=number) + return {str(i.get("hostWwn", "")).lower() for i in items} + items = self._data("/host-iscsis", portId=port, hostGroupNumber=number) + return {str(i.get("iscsiName", "")).lower() for i in items} + + def _add_member(self, protocol: Protocol, port: str, number: int, address: str) -> None: + if protocol == Protocol.FC: + self._run("POST", "/host-wwns", + {"hostWwn": address, "portId": port, "hostGroupNumber": number}) + else: + self._run("POST", "/host-iscsis", + {"iscsiName": address, "portId": port, "hostGroupNumber": number}) + + def ensure_host(self, spec: HostSpec) -> str: + wanted = {p: spec.addresses(p) for p in (Protocol.FC, Protocol.ISCSI)} + wanted = {p: a for p, a in wanted.items() if a} + plan: list[tuple[Protocol, str, list[dict[str, Any]]]] = [] + for protocol, addresses in wanted.items(): + ports = self._ports_for(protocol) + if not ports: + raise ArrayError(f"Hitachi VSP array {self.config.id} has no configured " + f"{protocol.value} port (option 'ports': {self._ports})") + for port in ports: + groups = self._groups(port) + # Ownership check before any change: FC refuses a WWN in two + # host groups of a port, iSCSI does not, so scan both. + for g in groups: + if g.get("hostGroupName") == spec.name: + continue + taken = self._members(protocol, port, int(g["hostGroupNumber"])) + clash = sorted(set(addresses) & taken) + if clash: + raise ArrayConflictError( + f"Initiator {clash[0]} is registered to host group " + f"{g.get('hostGroupName')!r} ({g.get('hostGroupId')}) " + f"on port {port}") + plan.append((protocol, port, groups)) + with self._change(): + for protocol, port, groups in plan: + current = self._by_name(groups, spec.name) + if current is None: + body: dict[str, Any] = {"portId": port, "hostGroupName": spec.name, + "hostMode": self.option("host_mode")} + if self.option("host_mode_options"): + body["hostModeOptions"] = [int(o) for o in + self.option("host_mode_options")] + affected = self._run("POST", "/host-groups", body) + ids = [_tail(r) for r in affected if "/host-groups/" in r] + if ids: + number = int(ids[0].split(",")[1]) + else: + found = self._by_name(self._groups(port), spec.name) + if found is None: + raise ArrayRequestError( + f"Host group {spec.name} not found on {port} after create") + number = int(found["hostGroupNumber"]) + have: set[str] = set() + else: + number = int(current["hostGroupNumber"]) + have = self._members(protocol, port, number) + for address in wanted[protocol]: + if address.lower() not in have: + self._add_member(protocol, port, number, address) + return spec.name + + def delete_host(self, host_id: str) -> None: + targets = [(p, n) for p, n in self._host_groups(host_id) if n != 0] + if not targets: + return + with self._change(): + for port, number in targets: + try: + self._run("DELETE", f"/host-groups/{port},{number}") + except ArrayNotFoundError: + pass + + # -- mappings -------------------------------------------------------------------- + + def _paths(self, ldev: dict[str, Any], host_id: str | None = None + ) -> list[dict[str, Any]]: + order = {p: i for i, p in enumerate(self._ports)} + paths = [p for p in ldev.get("ports") or [] + if p.get("portId") in order + and (host_id is None or p.get("hostGroupName") == host_id)] + return sorted(paths, key=lambda p: order[p["portId"]]) + + def _mappings(self, ldev: dict[str, Any], host_id: str | None = None) -> list[Mapping]: + out: dict[str, Mapping] = {} + for p in self._paths(ldev, host_id): + name = str(p.get("hostGroupName")) + out.setdefault(name, Mapping(volume_id=str(ldev["ldevId"]), host_id=name, + lun=int(p["lun"]))) + return list(out.values()) + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + ldev = self._require(volume_id) + groups = self._host_groups(host_id) + if not groups: + raise ArrayNotFoundError(f"Host {host_id} has no host group on " + f"ports {', '.join(self._ports)}") + present = {(p["portId"], int(p["hostGroupNumber"])): int(p["lun"]) + for p in self._paths(ldev, host_id)} + missing = [g for g in groups if g not in present] + if missing: + used: set[int] = set() + for port, number in missing: + used |= {int(lu["lun"]) for lu in + self._data("/luns", portId=port, hostGroupNumber=number)} + preferred = next(iter(present.values()), None) + lun = preferred if preferred is not None and preferred not in used else next( + (n for n in range(MAX_LUN + 1) if n not in used), None) + if lun is None: + raise ArrayConflictError(f"Host {host_id} has no free LUN") + with self._change(): + for port, number in missing: + self._run("POST", "/luns", {"portId": port, "hostGroupNumber": number, + "ldevId": int(ldev["ldevId"]), "lun": lun}) + ldev = self._require(volume_id) + mappings = self._mappings(ldev, host_id) + if not mappings: + raise ArrayRequestError(f"LDEV {volume_id} shows no path to {host_id} after mapping") + return mappings[0] + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + ldev = self._ldev(volume_id) + if ldev is None: + return + paths = self._paths(ldev, host_id) + if not paths: + return + with self._change(): + for p in paths: + try: + self._run("DELETE", + f"/luns/{p['portId']},{p['hostGroupNumber']},{p['lun']}") + except ArrayNotFoundError: + pass + + def list_mappings(self, volume_id: str) -> list[Mapping]: + return self._mappings(self._require(volume_id)) diff --git a/simplyblock_core/storage_arrays/adapters/hpe_nimble.py b/simplyblock_core/storage_arrays/adapters/hpe_nimble.py new file mode 100644 index 0000000000..e2036c5c98 --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/hpe_nimble.py @@ -0,0 +1,403 @@ +"""HPE Alletra 5000/6000 and Nimble Storage, NimbleOS REST API v1. + +The endpoint normally includes the API port: ``https://array:5392``. + +Auth: ``POST /v1/tokens {"data": {"username", "password"}}`` returns +``data.session_token`` (sent as ``X-Auth-Token``) and ``data.id`` (used to +release the token with ``DELETE /v1/tokens/``). Every request and +response body is wrapped in ``{"data": ...}``; errors are +``{"messages": [{"code": "SM_...", "severity": "error", "text": "..."}]}``. + +Volumes: sizes are in MiB. ``Volume.id`` is the NimbleOS object id. A volume +must be taken offline (``PUT {"data": {"online": false}}``) before it can be +deleted. + +Hosts are *initiator groups* (``access_protocol`` ``iscsi`` or ``fc``); the +host id is the group id. Initiators are added one by one with ``POST +/v1/initiators``; FC WWPNs use the colon form. Before adding, the adapter +looks each initiator up and raises :class:`ArrayConflictError` if another +group owns it (whether NimbleOS itself forbids an initiator in two groups is +not relied on). Mappings are *access control records* (ACRs) with +``apply_to: "both"`` (volume and its snapshots); the array assigns the LUN. + +Volume identity: NimbleOS volumes report a 32-hex ``serial_number`` and +present it to hosts as a 16-byte **EUI-64** designator (multipath WWID +``2``, sysfs ``wwid`` ``eui.``), not an NAA. The adapter +reports it as ``wwn="eui."`` (``Volume.wwn`` holds the SCSI +designator, NAA or EUI-64), so the identifier equals the host's sysfs wwid. + +Asynchronous jobs: a mutating request may answer **202** with +``{"messages": [{"code": "SM_async_job_id", "arguments": {"job_id": ...}}]}`` +instead of the object; the adapter then polls ``GET /v1/jobs/`` until +``state`` is ``done`` (as the NimbleOS SDK does) and re-reads what it needs. + +Checked against vendor source +----------------------------- +(HPE ``nimble-python-sdk``: ``nimbleclient/v1/restclient.py``, the +``nimbleclient/v1/api/*.py`` field docs and ``tests/``; Cinder +``cinder/volume/drivers/hpe/nimble.py``; HPE ``common-host-libs`` +``linux/device.go``.) + +* Tokens: ``POST /v1/tokens {"data": {username, password}}`` -> + ``data.session_token`` / ``data.id``; ``X-Auth-Token``; release with + ``DELETE /v1/tokens/``; bad logins and expired tokens answer + ``SM_http_unauthorized``. +* ``{"data": ...}`` envelopes; errors ``{"messages": [...]}`` where a generic + ``SM_http_`` entry precedes the specific one (Cinder reads + ``messages[1]``). Specific codes seen in vendor code: ``SM_eexist`` (with + ``SM_http_conflict``) for duplicates, ``SM_enoent`` / ``SM_http_not_found`` + for unknown objects, ``SM_vol_size_decreased`` (shrink without ``force``; + we never send ``force``), ``SM_vol_has_clone``. Older NimbleOS ("fiji and + below") answers ``SM_http_bad_request`` (400) for an unknown id, hence + ``get_volume`` treats 400 as missing. +* Sizes in MiB; ``serial_number`` is the volume's SCSI identifier; arrays + report ``model``/``serial``/``version``. +* Offline before delete (``PUT {"online": false}``). Offlining a volume that + is already offline fails with "Object is already in requested state"; + ``delete_volume`` tolerates that, as Cinder does, so a retried delete works. +* Query filters as plain parameters (``?vol_id=...``, + ``?initiator_group_id=...``, ``?name=...``) returning an empty ``data`` + list when nothing matches (Cinder ``get_query`` callers). +* ``label`` is required for iSCSI initiators; FC WWPNs in colon form (Cinder + ``_format_to_wwpn``); ACRs with ``apply_to: "both"``. +* ACR ``lun``: 0 for iSCSI, 0-2047 for FC (SDK field doc); Cinder uses LUN 0 + unless group-scoped targets are enabled. +* Multipath shows an EUI-64 device as WWID ``2``: common-host-libs + strips the leading type digit (2 EUI, 3 NAA) to compare with the serial + the CSP returns. + +Verify against a real system +---------------------------- +* That this CSP serial is the volume's ``serial_number`` on NimbleOS 5.x/6.x + for iSCSI and FC, i.e. ``eui.`` + lowercase ``serial_number`` equals the + sysfs ``wwid`` (the nimble-csp source is not published). +* Status and code of the refusals the fake invents: deleting an online + volume (``SM_vol_online``?), offlining or deleting a volume that has ACRs, + deleting an initiator group with ACRs, offlining an offline volume. +* Which operations answer 202 on a real array, and the ``result`` of a + failed job (the SDK only waits for ``state == "done"``). +* ``label`` length limit (64?) and uniqueness scope; whether an initiator may + sit in two groups (not relied on: the adapter checks ownership first). +""" +from __future__ import annotations + +import hashlib +import math +from typing import Any +from urllib.parse import quote + +import requests + +from ..base import ( + ArrayAdapter, + ArrayAuthError, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, + colon_wwpn, + normalize_wwpn, +) +from ..http import body_text, json_of + +MIB = 1 << 20 +_JOB_POLL_S = 2.0 +#: markers of "Object is already in requested state" (offlining an offline volume) +_ALREADY = ("SM_ealready", "already in requested state") +_NOT_FOUND_CODES = frozenset({"SM_enoent", "SM_http_not_found", "SM_no_such_object"}) + + +def nimble_eui(serial: str) -> str: + return "eui." + serial.lower() + + +class HpeNimbleAdapter(ArrayAdapter): + vendor = "hpe-nimble" + display_name = "HPE Alletra 5000/6000 / Nimble" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC}), + size_granularity_bytes=MIB, + min_size_bytes=MIB, + max_name_len=63, + ) + credential_fields = (frozenset({"username", "password"}),) + option_docs = ( + OptionDoc("pool", "Pool name new volumes are created in (default: the " + "array's default pool)."), + OptionDoc("performance_policy", "Performance policy name for new volumes " + "(default: the array default)."), + OptionDoc("job_timeout_s", "How long to wait for an asynchronous (HTTP 202) job.", + default=300), + ) + + def __init__(self, ctx): + super().__init__(ctx) + self._token_id: str | None = None + self._create_extra: dict[str, str] | None = None + + # -- session ------------------------------------------------------------ + + def login(self) -> None: + self.session.headers.pop("X-Auth-Token", None) + data = self._data("POST", "/v1/tokens", + json={"data": {"username": self.secret("username"), + "password": self.secret("password")}}) + token = (data or {}).get("session_token") + if not token: + raise ArrayRequestError("NimbleOS login returned no session_token") + self._token_id = data.get("id") + self.session.headers["X-Auth-Token"] = token + + def close(self) -> None: + token_id, self._token_id = self._token_id, None + if token_id and "X-Auth-Token" in self.session.headers: + try: + self.session.delete(f"/v1/tokens/{quote(token_id, safe='')}", + ok=(401, 404)) + except Exception: + pass + self.session.headers.pop("X-Auth-Token", None) + + # -- request helpers ----------------------------------------------------- + + def _request(self, method: str, path: str, *, params: Any = None, json: Any = None, + missing_ok: bool = False) -> requests.Response | None: + resp = self.session.request(method, path, params=params, json=json, + ok=(400, 401, 403, 404, 409)) + status = resp.status_code + if status < 400: + return resp + codes: list[str] = [] + text = body_text(resp) + try: + messages = (resp.json() or {}).get("messages") or [] + codes = [str(m.get("code", "")) for m in messages] + text = "; ".join(f"{m.get('code', '')}: {m.get('text', '')}" for m in messages) or text + except (ValueError, AttributeError): + pass + msg = f"{method} {path} -> HTTP {status}: {text}" + if status in (401, 403): + raise ArrayAuthError(msg) + if status == 404 or any(c in _NOT_FOUND_CODES for c in codes): + if missing_ok: + return None + raise ArrayNotFoundError(msg) + if status == 409 or "SM_eexist" in codes: + raise ArrayConflictError(msg) + raise ArrayRequestError(msg, status=status) + + def _data(self, method: str, path: str, **kw: Any) -> Any: + """The response's ``data``; ``None`` for a missing object (with + ``missing_ok``) or after waiting for an asynchronous job, in which + case the caller re-reads the object.""" + resp = self._request(method, path, **kw) + if resp is None: + return None + body = json_of(resp) or {} + if resp.status_code == 202: + self._wait_job(body, f"{method} {path}") + return None + return body.get("data") + + def _wait_job(self, body: dict[str, Any], what: str) -> None: + job_id = None + for m in body.get("messages") or []: + if m.get("code") == "SM_async_job_id": + job_id = (m.get("arguments") or {}).get("job_id") + if not job_id: + return # 202 without a job: nothing to wait for + polls = max(1, math.ceil(float(self.option("job_timeout_s")) / _JOB_POLL_S)) + for _ in range(polls): + job = self._data("GET", f"/v1/jobs/{quote(str(job_id), safe='')}") or {} + if job.get("state") == "done": + return + self.session.sleep(_JOB_POLL_S) + raise ArrayRequestError(f"{what}: job {job_id} did not finish within " + f"{self.option('job_timeout_s')} s") + + def _list(self, path: str, **filters: str) -> list[dict[str, Any]]: + return list(self._data("GET", path, params=filters) or []) + + @staticmethod + def _volume(item: dict[str, Any]) -> Volume: + serial = item.get("serial_number") or "" + return Volume(id=item["id"], name=item["name"], size_bytes=int(item["size"]) * MIB, + wwn=nimble_eui(serial) if serial else None) + + def _lookup_id(self, path: str, name: str, what: str) -> str: + items = self._list(path, name=name) + if not items: + raise ArrayError(f"{self.display_name} array {self.config.id}: " + f"{what} {name!r} not found") + return str(items[0]["id"]) + + def _volume_extra(self) -> dict[str, str]: + if self._create_extra is None: + extra: dict[str, str] = {} + if self.option("pool"): + extra["pool_id"] = self._lookup_id("/v1/pools/detail", self.option("pool"), + "pool") + if self.option("performance_policy"): + extra["perfpolicy_id"] = self._lookup_id( + "/v1/performance_policies/detail", self.option("performance_policy"), + "performance policy") + self._create_extra = extra + return self._create_extra + + # -- contract ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + arrays = self._list("/v1/arrays/detail") + arr = arrays[0] if arrays else {} + return ArrayInfo(vendor=self.vendor, model=str(arr.get("model", "")), + version=str(arr.get("version", "")), + serial=str(arr.get("serial", ""))) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + body = {"name": name, "size": -(-size_bytes // MIB), **self._volume_extra()} + data = self._data("POST", "/v1/volumes", json={"data": body}) + vol = self._volume(data) if data else self.find_volume(name) + if vol is None: + raise ArrayRequestError(f"Volume {name} not found right after creation") + return vol + + def get_volume(self, volume_id: str) -> Volume | None: + try: + data = self._data("GET", f"/v1/volumes/{quote(volume_id, safe='')}", + missing_ok=True) + except ArrayRequestError as e: + if e.status == 400: # malformed id: cannot exist + return None + raise + return self._volume(data) if data else None + + def find_volume(self, name: str) -> Volume | None: + items = [i for i in self._list("/v1/volumes/detail", name=name) if i.get("name") == name] + return self._volume(items[0]) if items else None + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + data = self._data("PUT", f"/v1/volumes/{quote(volume_id, safe='')}", + json={"data": {"size": -(-size_bytes // MIB)}}) + if data: + return self._volume(data) + vol = self.get_volume(volume_id) + if vol is None: + raise ArrayNotFoundError(f"Volume {volume_id} vanished during resize") + return vol + + def delete_volume(self, volume_id: str) -> None: + path = f"/v1/volumes/{quote(volume_id, safe='')}" + try: + self._data("PUT", path, json={"data": {"online": False}}) + except (ArrayRequestError, ArrayConflictError) as e: + # A retried delete finds the volume already offline. + if not any(marker in str(e) for marker in _ALREADY): + raise + self._data("DELETE", path) + + # -- hosts (initiator groups) ------------------------------------------- + + def _group(self, name: str) -> dict[str, Any] | None: + items = [g for g in self._list("/v1/initiator_groups/detail", name=name) + if g.get("name") == name] + return items[0] if items else None + + def _owner(self, protocol: str, key: str, value: str) -> str | None: + """Name of the initiator group that owns ``value`` (iqn or wwpn).""" + for i in self._list("/v1/initiators/detail", access_protocol=protocol, **{key: value}): + have = str(i.get(key, "")) + same = (normalize_wwpn(have) == normalize_wwpn(value) if key == "wwpn" + else have.lower() == value.lower()) + if same: + return str(i.get("initiator_group_name") or i.get("initiator_group_id")) + return None + + def ensure_host(self, spec: HostSpec) -> str: + unsupported = [p for p in spec.protocols if not p.is_scsi] + if unsupported: + raise ArrayError(f"{self.display_name} adapter does not support " + + ", ".join(sorted(p.value for p in unsupported))) + iqns = spec.addresses(Protocol.ISCSI) + wwpns = [colon_wwpn(w) for w in spec.addresses(Protocol.FC)] + if iqns and wwpns: + raise ArrayError(f"{self.display_name}: an initiator group is either iSCSI or " + "FC; register the host once per protocol") + protocol, key, wanted = ("iscsi", "iqn", iqns) if iqns else ("fc", "wwpn", wwpns) + + group = self._group(spec.name) + if group is not None and group.get("access_protocol") != protocol: + raise ArrayConflictError( + f"Initiator group {spec.name} exists with access_protocol " + f"{group.get('access_protocol')}, not {protocol}") + if group is not None: + list_key = "iscsi_initiators" if protocol == "iscsi" else "fc_initiators" + have = {str(i.get(key, "")).lower() for i in group.get(list_key) or []} + wanted = [w for w in wanted if w.lower() not in have] + for value in wanted: + owner = self._owner(protocol, key, value) + if owner is not None and owner != spec.name: + raise ArrayConflictError( + f"Initiator {value} already belongs to initiator group {owner}") + if group is None: + group = self._data("POST", "/v1/initiator_groups", + json={"data": {"name": spec.name, "access_protocol": protocol, + "description": "simplyblock"}}) \ + or self._group(spec.name) + if group is None: + raise ArrayRequestError(f"Initiator group {spec.name} not found right " + "after creation") + group_id = str(group["id"]) + for value in wanted: + body: dict[str, Any] = {"initiator_group_id": group_id, + "access_protocol": protocol, key: value} + if protocol == "iscsi": + body["label"] = f"sb-{hashlib.sha1(value.encode()).hexdigest()[:12]}" + self._data("POST", "/v1/initiators", json={"data": body}) + return group_id + + def delete_host(self, host_id: str) -> None: + self._data("DELETE", f"/v1/initiator_groups/{quote(host_id, safe='')}", + missing_ok=True) + + # -- mappings (access control records) ----------------------------------- + + def _acrs(self, volume_id: str, host_id: str | None = None) -> list[dict[str, Any]]: + items = self._list("/v1/access_control_records/detail", vol_id=volume_id) + return [a for a in items if a.get("vol_id") == volume_id and a.get("initiator_group_id") + and (host_id is None or a.get("initiator_group_id") == host_id)] + + @staticmethod + def _mapping(acr: dict[str, Any]) -> Mapping: + lun = acr.get("lun") + return Mapping(volume_id=acr["vol_id"], host_id=acr["initiator_group_id"], + lun=int(lun) if lun is not None else None) + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + existing = self._acrs(volume_id, host_id) + if existing: + return self._mapping(existing[0]) + acr = self._data("POST", "/v1/access_control_records", + json={"data": {"vol_id": volume_id, "initiator_group_id": host_id, + "apply_to": "both"}}) + if acr: + return self._mapping(acr) + existing = self._acrs(volume_id, host_id) + if not existing: + raise ArrayRequestError(f"ACR {volume_id} -> {host_id} not found after creation") + return self._mapping(existing[0]) + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + for acr in self._acrs(volume_id, host_id): + self._data("DELETE", + f"/v1/access_control_records/{quote(str(acr['id']), safe='')}", + missing_ok=True) + + def list_mappings(self, volume_id: str) -> list[Mapping]: + return [self._mapping(a) for a in self._acrs(volume_id)] diff --git a/simplyblock_core/storage_arrays/adapters/hpe_wsapi.py b/simplyblock_core/storage_arrays/adapters/hpe_wsapi.py new file mode 100644 index 0000000000..5cff6bf3d6 --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/hpe_wsapi.py @@ -0,0 +1,350 @@ +"""HPE WSAPI: Alletra Storage MP B10000, Primera and 3PAR StoreServ. + +Auth: ``POST /api/v1/credentials {"user", "password"}`` returns ``{"key"}``, +sent as the ``X-HP3PAR-WSAPI-SessionKey`` header on every later request and +released with ``DELETE /api/v1/credentials/``. An expired or unknown key +is answered with HTTP 401/403 (mapped to :class:`ArrayAuthError`, so the client +logs in again). + +Objects are addressed by name: ``Volume.id`` is the volume name and the host +id is the host name. Errors carry ``{"code": , "desc": "..."}``. + +Volumes: ``POST /api/v1/volumes {name, cpg, sizeMiB, tpvv|reduce}``; grow is +``PUT /api/v1/volumes/ {"action": 3, "sizeMiB": }`` -- GROW +takes the *delta*, not the new size. SCSI identity: ``naa.`` from the +volume's ``wwn`` field (``60002AC...``). + +Hosts: ``POST /api/v1/hosts {name, persona, iSCSINames|FCWWNs}``; paths are +added with ``PUT /api/v1/hosts/ {"pathOperation": 1, ...}``. Note the +asymmetry: requests use ``iSCSINames``/``FCWWNs``, responses report +``iSCSIPaths[].name``/``FCPaths[].wwn``. + +VLUNs (mappings): ``POST /api/v1/vluns {volumeName, hostname, lun, autoLun, +maxAutoLun}`` answers 201 with an empty body and a ``Location`` header +``/api/v1/vluns/,,``. ``GET /api/v1/vluns?query="volumeName EQ +"`` lists both the VLUN *template* and one *active* VLUN per host path, +so entries are de-duplicated by (volume, host). + +Only iSCSI and FC hosts are supported: NVMe/TCP and NVMe/FC hosts exist on +Alletra MP and Primera (python-3parclient's ``createHost`` sends an ``NQN`` +field) but NVMe host paths and exports are not implemented here, so NVMe is +refused up front. + +Checked against vendor source +----------------------------- +(python-3parclient ``hpe3parclient/client.py``/``http.py`` and its WSAPI mock +``test/HPE3ParMockServer_flask.py``; HPE ``hpe_storage_flowkit_py`` +``v1/core/rest_client.py``/``session.py``/``utils/volume_utils.py``, the +library behind Cinder's Alletra MP driver; Cinder ``hpe_3par_common.py``, +``hpe_3par_iscsi.py``; HPE ``common-host-libs`` ``linux/device.go``.) + +* Login/logout, the session header, ``{"code", "desc"}`` errors. +* Error codes: 5 INV_USER_PASS (403 on login), 16 EXISTENT_HOST (409), 17 + NON_EXISTENT_HOST (404), 19 NON_EXISTENT_VLUN (404), 22 EXISTENT_VOL + (409), 23 NON_EXISTENT_VOL (404), 26 EXPORTED_VLUN (409, host still + exported), 34 "resource in use" (409, e.g. volume in a VV set or + exported), 73 EXISTENT_PATH (409). Code 29 is *not* a VLUN conflict: + Cinder sees it as HTTP 400 on a volume delete, so it is no longer treated + as a conflict. +* 403 is also used for non-auth refusals (PERM_DENIED, 150 cannot grow this + volume type, 153 new size exceeds the CPG limit, 120 volume in remote copy, + VV_SIZE_CANNOT_REDUCE, ...). python-3parclient still re-authenticates once + on any 401/403, so a 403 stays an auth error except for the listed + non-auth codes (``_FORBIDDEN_NOT_AUTH``), which fail directly. +* GROW (``action`` 3) takes the increment in MiB; minimum create size is + 256 MiB, name limit 31 characters (vendor mock). +* ``reduce: true`` is the dedup+compression create key on Primera *and* + Alletra MP (python-3parclient maps ``tdvv``+``compression`` to it when the + WSAPI is Primera-class; flowkit sends ``tpvv``/``reduce``); ``tpvv: true`` + is thin. 3PAR (pre-Primera) dedup/compression is not offered. +* ``POST /hosts`` and ``PUT /hosts`` refuse ``iSCSINames`` together with + ``FCWWNs`` (INV_INPUT_PARAM_CONFLICT), hence one path type per request; a + host may carry both types (Cinder's iSCSI driver adds iSCSI names to an + existing host that has none). +* VLUN query ``"volumeName EQ "`` (quotes included), ``members`` list, + template VLUNs ``active: false`` plus active per-path VLUNs; the + ``Location`` header is ``/api/v1/vluns/,,``. +* Personas: 1 Generic, 2 Generic-ALUA, 8 VMware, 11 WindowsServer. +* Volume ``wwn`` field; HPE host tooling shows 3PAR-family WWIDs as + ``naa.60002ac...``. + +Verify against a real system +---------------------------- +* The status/code for an expired or unknown session key (the fake uses 401 + code 6); python-3parclient re-authenticates on both 401 and 403. +* Size granularity on Alletra MP (the 256 MiB step is from the vendor mock + and growVolume's "rounded up to the next chunklet size (e.g. 256 or 1000 + MiB)"); we round to 256 MiB so requested and reported sizes agree. +* A filtered VLUN query with no hits answering 200 with an empty + ``members`` list (python-3parclient implies it but cannot tell it apart + from a 404) and the status/code for a duplicate VLUN. +* Max session count per user (stale keys from crashed processes may exhaust + it; flowkit reuses one session per user and drops it after 14 minutes); + we always ``DELETE`` the key on close. +""" +from __future__ import annotations + +import re +from typing import Any +from urllib.parse import quote, unquote + +import requests + +from ..base import ( + ArrayAdapter, + ArrayAuthError, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, +) +from ..http import body_text, json_of + +MIB = 1 << 20 +_GRANULARITY = 256 * MIB + +SESSION_HEADER = "X-HP3PAR-WSAPI-SessionKey" +_API = "/api/v1" + +GROW_VOLUME = 3 +PATH_ADD = 1 + +_NOT_FOUND_CODES = frozenset({17, 19, 23}) +_CONFLICT_CODES = frozenset({16, 22, 26, 34, 73}) +#: 403 codes that are refusals, not session/credential problems: 120 volume +#: in remote copy, 150 cannot grow this volume type, 153 new size exceeds the +#: CPG limit, 215 remote copy group already started. +_FORBIDDEN_NOT_AUTH = frozenset({120, 150, 153, 215}) + +_LOCATION_RE = re.compile(r"/api/v1/vluns/(?P[^,]+),(?P\d+),(?P[^,/]*)") + + +def wsapi_wwn(wwn: str) -> str: + return "naa." + wwn.lower() + + +class HpeWsapiAdapter(ArrayAdapter): + vendor = "hpe-wsapi" + display_name = "HPE Alletra MP / Primera / 3PAR" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC}), + size_granularity_bytes=_GRANULARITY, + min_size_bytes=_GRANULARITY, + max_name_len=31, + ) + credential_fields = (frozenset({"username", "password"}),) + option_docs = ( + OptionDoc("cpg", "Common Provisioning Group new volumes are created in.", + required=True), + OptionDoc("provisioning", "'tpvv' (thin) or 'reduce' (thin + dedup + " + "compression, Primera / Alletra MP).", default="tpvv"), + OptionDoc("persona", "Host persona id for new hosts (2 = Generic-ALUA).", + default=2), + ) + + def __init__(self, ctx): + super().__init__(ctx) + self._key: str | None = None + provisioning = str(self.option("provisioning")).lower() + if provisioning not in ("tpvv", "reduce"): + raise ArrayError(f"{self.display_name}: provisioning must be 'tpvv' or " + f"'reduce', not {provisioning!r}") + self._provisioning = provisioning + + # -- session ------------------------------------------------------------ + + def login(self) -> None: + self.session.headers.pop(SESSION_HEADER, None) + resp = self._call("POST", "/credentials", + json={"user": self.secret("username"), + "password": self.secret("password")}) + key = (json_of(resp) or {}).get("key") + if not key: + raise ArrayRequestError("WSAPI login returned no session key") + self._key = key + self.session.headers[SESSION_HEADER] = key + + def close(self) -> None: + key, self._key = self._key, None + self.session.headers.pop(SESSION_HEADER, None) + if key: + try: + self.session.delete(f"{_API}/credentials/{quote(key, safe='')}", + headers={SESSION_HEADER: key}, ok=(401, 403, 404)) + except Exception: + pass + + # -- request helpers ----------------------------------------------------- + + def _call(self, method: str, path: str, *, params: Any = None, json: Any = None, + missing_ok: bool = False) -> requests.Response: + """One WSAPI request; maps ``{"code", "desc"}`` errors to ArrayErrors. + With ``missing_ok`` a not-found answer is returned instead of raised.""" + resp = self.session.request(method, f"{_API}{path}", params=params, json=json, + ok=(400, 401, 403, 404, 409)) + status = resp.status_code + if status < 400: + return resp + code: int | None = None + desc = body_text(resp) + try: + body = resp.json() + if isinstance(body, dict): + code = int(body["code"]) if body.get("code") is not None else None + desc = str(body.get("desc", desc)) + except (ValueError, TypeError, KeyError): + pass + msg = f"{method} {path} -> HTTP {status}" + (f" code {code}" if code is not None else "") \ + + f": {desc}" + if status == 403 and code in _FORBIDDEN_NOT_AUTH: + raise ArrayRequestError(msg, status=status) + if status in (401, 403): + raise ArrayAuthError(msg) + if status == 404 or code in _NOT_FOUND_CODES: + if missing_ok: + return resp + raise ArrayNotFoundError(msg) + if status == 409 or code in _CONFLICT_CODES: + raise ArrayConflictError(msg) + raise ArrayRequestError(msg, status=status) + + @staticmethod + def _found(resp: requests.Response) -> bool: + return resp.status_code < 400 + + @staticmethod + def _volume(item: dict[str, Any]) -> Volume: + wwn = item.get("wwn") or "" + return Volume(id=item["name"], name=item["name"], + size_bytes=int(item["sizeMiB"]) * MIB, + wwn=wsapi_wwn(wwn) if wwn else None) + + # -- contract ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + system = json_of(self._call("GET", "/system")) or {} + return ArrayInfo(vendor=self.vendor, model=str(system.get("model", "")), + version=str(system.get("systemVersion", "")), + serial=str(system.get("serialNumber", ""))) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + body: dict[str, Any] = {"name": name, "cpg": self.option("cpg"), + "sizeMiB": -(-size_bytes // MIB)} + body["reduce" if self._provisioning == "reduce" else "tpvv"] = True + self._call("POST", "/volumes", json=body) + vol = self.get_volume(name) + if vol is None: + raise ArrayRequestError(f"Volume {name} not found right after creation") + return vol + + def get_volume(self, volume_id: str) -> Volume | None: + resp = self._call("GET", f"/volumes/{quote(volume_id, safe='')}", missing_ok=True) + return self._volume(json_of(resp)) if self._found(resp) else None + + def find_volume(self, name: str) -> Volume | None: + return self.get_volume(name) + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + vol = self.get_volume(volume_id) + if vol is None: + raise ArrayNotFoundError(f"Volume {volume_id} does not exist") + delta = -(-size_bytes // MIB) - vol.size_bytes // MIB + if delta > 0: + self._call("PUT", f"/volumes/{quote(volume_id, safe='')}", + json={"action": GROW_VOLUME, "sizeMiB": delta}) + elif delta < 0: + raise ArrayConflictError(f"WSAPI cannot shrink volume {volume_id}") + new = self.get_volume(volume_id) + if new is None: + raise ArrayNotFoundError(f"Volume {volume_id} vanished during resize") + return new + + def delete_volume(self, volume_id: str) -> None: + self._call("DELETE", f"/volumes/{quote(volume_id, safe='')}") + + def _host(self, name: str) -> dict[str, Any] | None: + resp = self._call("GET", f"/hosts/{quote(name, safe='')}", missing_ok=True) + return json_of(resp) if self._found(resp) else None + + def ensure_host(self, spec: HostSpec) -> str: + unsupported = [p for p in spec.protocols if not p.is_scsi] + if unsupported: + raise ArrayError(f"{self.display_name} adapter does not support " + + ", ".join(sorted(p.value for p in unsupported))) + iqns = spec.addresses(Protocol.ISCSI) + wwns = [w.upper() for w in spec.addresses(Protocol.FC)] + host = self._host(spec.name) + if host is None: + # Create with one path type (WSAPI refuses iSCSINames together + # with FCWWNs: INV_INPUT_PARAM_CONFLICT); a second type is added below. + body: dict[str, Any] = {"name": spec.name, "persona": int(self.option("persona"))} + if iqns: + body["iSCSINames"] = iqns + else: + body["FCWWNs"] = wwns + self._call("POST", "/hosts", json=body) + if not (iqns and wwns): + return spec.name + host = {"iSCSIPaths": [{"name": i} for i in iqns], "FCPaths": []} + have_iqns = {str(p.get("name", "")).lower() for p in host.get("iSCSIPaths") or []} + have_wwns = {str(p.get("wwn", "")).upper() for p in host.get("FCPaths") or []} + add_iqns = [i for i in iqns if i.lower() not in have_iqns] + add_wwns = [w for w in wwns if w not in have_wwns] + # One path type per request: a modify with both is refused too. + if add_iqns: + self._call("PUT", f"/hosts/{quote(spec.name, safe='')}", + json={"pathOperation": PATH_ADD, "iSCSINames": add_iqns}) + if add_wwns: + self._call("PUT", f"/hosts/{quote(spec.name, safe='')}", + json={"pathOperation": PATH_ADD, "FCWWNs": add_wwns}) + return spec.name + + def delete_host(self, host_id: str) -> None: + self._call("DELETE", f"/hosts/{quote(host_id, safe='')}", missing_ok=True) + + def _vluns(self, volume_id: str, host_id: str | None = None) -> list[Mapping]: + resp = self._call("GET", "/vluns", params={"query": f'"volumeName EQ {volume_id}"'}, + missing_ok=True) + if not self._found(resp): + return [] + seen: dict[tuple[str, str], Mapping] = {} + for m in (json_of(resp) or {}).get("members", []): + host = m.get("hostname") + if not host or m.get("volumeName") != volume_id: + continue + if host_id is not None and host != host_id: + continue + # The template and every active per-path VLUN share (vol, lun, host). + seen.setdefault((volume_id, host), + Mapping(volume_id=volume_id, host_id=host, lun=int(m["lun"]))) + return list(seen.values()) + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + existing = self._vluns(volume_id, host_id) + if existing: + return existing[0] + resp = self._call("POST", "/vluns", + json={"volumeName": volume_id, "hostname": host_id, "lun": 0, + "autoLun": True, "maxAutoLun": 0}) + m = _LOCATION_RE.search(unquote(resp.headers.get("Location", ""))) + if m: + return Mapping(volume_id=volume_id, host_id=host_id, lun=int(m["lun"])) + existing = self._vluns(volume_id, host_id) + if not existing: + raise ArrayRequestError(f"VLUN {volume_id} -> {host_id} not found after creation") + return existing[0] + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + for m in self._vluns(volume_id, host_id): + self._call("DELETE", "/vluns/" + quote(f"{volume_id},{m.lun},{host_id}", safe=","), + missing_ok=True) + + def list_mappings(self, volume_id: str) -> list[Mapping]: + return self._vluns(volume_id) diff --git a/simplyblock_core/storage_arrays/adapters/huawei_dorado.py b/simplyblock_core/storage_arrays/adapters/huawei_dorado.py new file mode 100644 index 0000000000..543f22933d --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/huawei_dorado.py @@ -0,0 +1,468 @@ +"""Huawei OceanStor Dorado (V6 / OceanStor 6.x) DeviceManager REST API. + +Auth: ``POST /deviceManager/rest/xx/sessions`` with ``{username, password, +scope: "0"}`` (scope 0 = local user) returns ``data.deviceid`` and +``data.iBaseToken`` (plus a session cookie, kept by the ``requests`` +session). Every later call goes to ``/deviceManager/rest//...`` +with the ``iBaseToken`` header. Logout is ``DELETE .../sessions``. + +Envelope: answers are HTTP 200 even on failure, with ``{"data": ..., +"error": {"code": , "description": "..."}}``; code 0 is success. A +query that matches nothing omits ``data``. All values are strings. Codes +mapped here (from Huawei's own CSI driver, eSDK_K8S_Plugin): + +* auth: -401 (not logged in / session expired), 1077949069 (user offline), + 1077987870 / 1077949081 / 1077949061 (bad user or password), + 1077949070 / 1077987871 (account locked), 1077949071 (IP locked) +* not found: 1077948996 (object), 1077936859 (LUN), 1077937498 (host), + 1077937500 (host group), 1077951819 (mapping view) +* conflict: 1077948993 (name exists), 1077948997 (id not unique) +* tolerated as "already done": 1077936862 (LUN already in LUN group), + 1077937501 (host already in host group), 1073745412 (host not in host + group), 1073804556 / 1073804560 (host/LUN group already in mapping view), + 1073804552 / 1073804554 (host/LUN group not in mapping view) + +LUNs: ``POST /lun {NAME, PARENTID: , CAPACITY: <512-byte sectors>, +ALLOCTYPE: 1}`` (thin; Dorado LUNs are always thin), ``PUT /lun/expand {ID, +CAPACITY}``, ``DELETE /lun/``. Identity: ``WWN`` -> ``naa.``. +The ``storage_pool`` option is a pool *name*, resolved once per session via +``GET /storagepool?filter=NAME::``. + +Hosts and initiators: ``POST /host {NAME, OPERATIONSYSTEM: 0}`` (Linux). +iSCSI/FC initiators are objects of their own (``/iscsi_initiator/``, +``/fc_initiator/``) with ``ISFREE`` / ``PARENTID``; one is created if +the array does not know it yet, then bound with ``PUT ... {PARENTTYPE: 21, +PARENTID: }``. An initiator whose ``ISFREE`` is ``"false"`` and +``PARENTID`` is another host raises :class:`ArrayConflictError`. Hosts are +addressed by id (``ensure_host`` returns it). + +Mapping: the classic host group + LUN group + mapping view triple, one per +host, as Huawei's CSI driver does: ``sb_hg_`` holds the host, +``sb_lg_`` holds the LUNs mapped to it, ``sb_mv_`` joins them. +Map/unmap add/remove the LUN to/from the host's LUN group; ``delete_host`` +tears the triple down, unbinds the initiators and deletes the host. The +host LUN id comes from ``GET /lun/associate?TYPE=11&ASSOCIATEOBJTYPE=21& +ASSOCIATEOBJID=`` (``HostLUNID`` inside the ``ASSOCIATEMETADATA`` JSON +string). ``list_mappings`` reports only mappings made through ``sb_lg_`` +groups; a LUN in a customer's LUN group is never touched (deleting it then +fails on the array). + +NVMe hosts are not supported by this adapter (iSCSI and FC only). + +Checked against vendor source +----------------------------- +Huawei CSI ``eSDK_K8S_Plugin`` (``storage/constants.go``, +``storage/oceanstorage/base/client*.go``, ``attacher/attacher.go``, +``oceanstor/client/client_lun.go``, ``csi/backend/plugin/oceanstor.go``) and +the OpenStack Cinder Huawei driver (``huawei/rest_client.py``, +``constants.py``, ``huawei_utils.py``): + +* Login/logout, ``deviceid``/``iBaseToken``, the envelope and every error + code listed above (``WrongPasswordErrorCodes``, ``AccountBeenLocked``, + ``NeedReLogin`` on -401 / 1077949069, the per-object constants). +* ``CAPACITY`` in 512-byte sectors (CSI sends ``ceil(bytes / 512)`` with no + further rounding; the 1 MiB rounding here is merely conservative); + ``ALLOCTYPE`` 1 = thin; ``PUT /lun/expand {ID, CAPACITY}``. +* Names: 31 characters (CSI ``maxLunNameLength``, Cinder + ``MAX_NAME_LENGTH``; V6 may allow more). +* ``filter=NAME::&range=[0-100]`` with a literal ``::`` (CSI uses the + first row as the exact match; only a value is ever escaped). The adapter + used to URL-encode the whole ``filter`` parameter (``NAME%3A%3A...``); it + now builds the query like the vendor drivers. +* Initiators: ``GET /iscsi_initiator/`` / ``/fc_initiator/`` + (1077948996 = unknown), ``ISFREE``/``PARENTID`` ownership as in CSI's + attacher, ``POST /iscsi_initiator {ID[, TYPE]}`` (1077948997 = exists), + bind with ``PUT .../ {PARENTTYPE: 21, PARENTID}``; unbind with ``PUT + /iscsi_initiator/remove_iscsi_from_host`` / ``/fc_initiator/ + remove_fc_from_host {TYPE, ID}``; list a host's with ``?PARENTTYPE=21& + PARENTID=`` -- Cinder re-checks ``PARENTID`` on each row and so does this + adapter now (an ignored filter would otherwise unbind every initiator on + the array). +* ``POST /host {NAME, OPERATIONSYSTEM: 0}``; host group / LUN group / + mapping view calls, ``DELETE`` with a JSON body (``/lungroup/associate``, + ``/host/associate``), ``PUT /mappingview/create_associate`` / + ``remove_associate``, ``GET /lun/associate?TYPE=11&ASSOCIATEOBJTYPE=21& + ASSOCIATEOBJID=`` with ``HostLUNID`` in ``ASSOCIATEMETADATA``. +* ``GET /system/``: ``ID``, ``PRODUCTMODE``, ``PRODUCTVERSION``, + ``productModeString``, ``pointRelease``. + +Verify against a real system +---------------------------- +* Whether ``CAPACITY`` stays in 512-byte units for 4K-sector LUNs + (``SECTORSIZE``). +* Creating an FC initiator object for a WWPN that has never logged in + (``POST /fc_initiator {ID, TYPE: "223"}``): neither vendor driver does it + (CSI skips unknown or offline WWPNs). +* Error codes for: deleting a LUN that is still in a LUN group, deleting a + host that still has initiators or group membership, binding an initiator + that is bound elsewhere, and a host already in another host group (a host + can belong to one host group only; the adapter checks this itself and + raises :class:`ArrayConflictError`). +* Some firmware may answer an expired session with HTTP 401 instead of the + -401 envelope (the vendor drivers only read the envelope); both are + treated as an auth error. +""" +from __future__ import annotations + +import json +from typing import Any +from urllib.parse import quote + +from ..base import ( + ArrayAdapter, + ArrayAuthError, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, +) +from ..http import json_of + +SECTOR = 512 +REST = "/deviceManager/rest" + +AUTH_CODES = frozenset({-401, 1077949069, 1077987870, 1077949081, 1077949061, + 1077949070, 1077987871, 1077949071}) +NOT_FOUND_CODES = frozenset({1077948996, 1077936859, 1077937498, 1077937500, 1077951819}) +CONFLICT_CODES = frozenset({1077948993, 1077948997}) + +LUN_ALREADY_IN_GROUP = 1077936862 +HOST_ALREADY_IN_GROUP = 1077937501 +HOST_NOT_IN_GROUP = 1073745412 +GROUP_ALREADY_IN_VIEW = frozenset({1073804556, 1073804560}) +GROUP_NOT_IN_VIEW = frozenset({1073804552, 1073804554}) + +OBJ_HOST = 21 +OBJ_HOSTGROUP = 14 +OBJ_LUN = 11 +OBJ_LUNGROUP = 256 +OBJ_MAPPINGVIEW = 245 +TYPE_ISCSI_INITIATOR = "222" +TYPE_FC_INITIATOR = "223" + +_LG, _HG, _MV = "sb_lg_", "sb_hg_", "sb_mv_" + + +def huawei_wwn(wwn: str) -> str: + return "naa." + wwn.lower() + + +class HuaweiDoradoAdapter(ArrayAdapter): + vendor = "huawei-dorado" + display_name = "Huawei OceanStor Dorado" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC}), + size_granularity_bytes=1 << 20, + min_size_bytes=1 << 20, + max_name_len=31, + ) + credential_fields = (frozenset({"username", "password"}),) + option_docs = ( + OptionDoc("storage_pool", "Name of the storage pool new LUNs are created in.", + required=True), + OptionDoc("scope", "Login scope: '0' local user, '1' LDAP user.", default="0"), + ) + + def __init__(self, ctx): + super().__init__(ctx) + self._base = "" + self._pool_id: str | None = None + + # -- session ----------------------------------------------------------------- + + def login(self) -> None: + self.session.headers.pop("iBaseToken", None) + self._base = "" + resp = self.session.post(f"{REST}/xx/sessions", json={ + "username": self.secret("username"), "password": self.secret("password"), + "scope": str(self.option("scope"))}) + data = self._unwrap("POST", "/xx/sessions", resp) + if not isinstance(data, dict) or not data.get("deviceid") or not data.get("iBaseToken"): + raise ArrayRequestError("OceanStor login returned no deviceid/iBaseToken") + self.session.headers["iBaseToken"] = data["iBaseToken"] + self._base = f"{REST}/{data['deviceid']}" + + def close(self) -> None: + if self._base and "iBaseToken" in self.session.headers: + try: + self.session.delete(f"{self._base}/sessions") + except Exception: + pass + + # -- request helpers ------------------------------------------------------------ + + @staticmethod + def _unwrap(method: str, path: str, resp, tolerate: frozenset[int] = frozenset(), + missing_ok: bool = False) -> Any: + body = json_of(resp) or {} + err = body.get("error") or {} + try: + code = int(err.get("code", 0)) + except (TypeError, ValueError): + code = -1 + if code == 0 or code in tolerate: + return body.get("data") + msg = f"{method} {path}: error {code}: {err.get('description', '')}".rstrip() + if err.get("suggestion"): + msg += f" ({err['suggestion']})" + if code in AUTH_CODES: + raise ArrayAuthError(msg) + if code in NOT_FOUND_CODES: + if missing_ok: + return None + raise ArrayNotFoundError(msg) + if code in CONFLICT_CODES: + raise ArrayConflictError(msg) + raise ArrayRequestError(msg) + + def _call(self, method: str, path: str, *, params: dict[str, Any] | None = None, + body: Any = None, tolerate: frozenset[int] = frozenset(), + missing_ok: bool = False) -> Any: + if not self._base: + raise ArrayAuthError("OceanStor session not established") + resp = self.session.request(method, f"{self._base}{path}", params=params, json=body) + return self._unwrap(method, path, resp, tolerate, missing_ok) + + def _list(self, path: str, params: dict[str, Any] | None = None) -> list[dict[str, Any]]: + data = self._call("GET", path, params=params) + return [d for d in data if isinstance(d, dict)] if isinstance(data, list) else [] + + @staticmethod + def _name_query(path: str, name: str) -> str: + """``?filter=NAME::&range=[0-100]`` with the ``::`` + separator and the range brackets left literal, as Huawei's CSI driver + and Cinder driver send them; only the name itself is percent-encoded.""" + return f"{path}?filter=NAME::{quote(name, safe='')}&range=[0-100]" + + def _by_name(self, path: str, name: str) -> dict[str, Any] | None: + rows = self._list(self._name_query(path, name)) + # The filter is treated as an exact match by the vendor drivers; check. + return next((r for r in rows if r.get("NAME") == name), None) + + # -- info / pool ------------------------------------------------------------------ + + def get_info(self) -> ArrayInfo: + s = self._call("GET", "/system/") or {} + return ArrayInfo( + vendor=self.vendor, + model=str(s.get("productModeString") or s.get("PRODUCTMODE") or ""), + version=str(s.get("pointRelease") or s.get("PRODUCTVERSION") or ""), + serial=str(s.get("ID", ""))) + + def _pool(self) -> str: + if self._pool_id is None: + name = str(self.option("storage_pool")) + pool = self._by_name("/storagepool", name) + if pool is None: + raise ArrayError(f"OceanStor storage pool {name!r} not found") + self._pool_id = str(pool["ID"]) + return self._pool_id + + # -- LUNs ----------------------------------------------------------------------------- + + @staticmethod + def _volume(lun: dict[str, Any]) -> Volume: + wwn = str(lun.get("WWN") or "") + return Volume(id=str(lun["ID"]), name=lun["NAME"], + size_bytes=int(lun["CAPACITY"]) * SECTOR, + wwn=huawei_wwn(wwn) if wwn else None) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + lun = self._call("POST", "/lun", body={ + "NAME": name, "PARENTID": self._pool(), "CAPACITY": size_bytes // SECTOR, + "ALLOCTYPE": 1, "DESCRIPTION": "simplyblock"}) + return self._volume(lun) + + def get_volume(self, volume_id: str) -> Volume | None: + lun = self._call("GET", f"/lun/{volume_id}", missing_ok=True) + return self._volume(lun) if lun else None + + def find_volume(self, name: str) -> Volume | None: + lun = self._by_name("/lun", name) + return self._volume(lun) if lun else None + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + self._call("PUT", "/lun/expand", body={"ID": volume_id, + "CAPACITY": size_bytes // SECTOR}) + vol = self.get_volume(volume_id) + if vol is None: + raise ArrayNotFoundError(f"LUN {volume_id} does not exist") + return vol + + def delete_volume(self, volume_id: str) -> None: + self._call("DELETE", f"/lun/{volume_id}") + + # -- hosts and initiators --------------------------------------------------------------- + + @staticmethod + def _initiator_kinds(spec: HostSpec) -> list[tuple[str, str, str]]: + """(collection, TYPE, address) for each initiator.""" + out = [("iscsi_initiator", TYPE_ISCSI_INITIATOR, a) + for a in spec.addresses(Protocol.ISCSI)] + out += [("fc_initiator", TYPE_FC_INITIATOR, a) for a in spec.addresses(Protocol.FC)] + return out + + def _initiator(self, collection: str, address: str) -> dict[str, Any] | None: + data = self._call("GET", f"/{collection}/{address}", missing_ok=True) + return data if isinstance(data, dict) and data else None + + def ensure_host(self, spec: HostSpec) -> str: + host = self._by_name("/host", spec.name) + host_id = str(host["ID"]) if host else None + # Check ownership before creating anything. + pending: list[tuple[str, str, str, bool]] = [] # collection, type, address, exists + for collection, itype, address in self._initiator_kinds(spec): + ini = self._initiator(collection, address) + if ini is not None and str(ini.get("ISFREE", "true")).lower() == "false": + parent = str(ini.get("PARENTID") or "") + if parent and parent != host_id: + raise ArrayConflictError( + f"Initiator {address} is registered to another host " + f"({ini.get('PARENTNAME') or parent})") + continue # already ours + pending.append((collection, itype, address, ini is not None)) + if host_id is None: + created = self._call("POST", "/host", body={ + "NAME": spec.name, "TYPE": OBJ_HOST, "OPERATIONSYSTEM": 0, + "DESCRIPTION": "simplyblock"}) + host_id = str(created["ID"]) + for collection, itype, address, exists in pending: + if not exists: + self._call("POST", f"/{collection}", body={"ID": address, "TYPE": itype}, + tolerate=frozenset({1077948997})) + self._call("PUT", f"/{collection}/{address}", + body={"TYPE": itype, "ID": address, + "PARENTTYPE": OBJ_HOST, "PARENTID": host_id}) + return host_id + + def delete_host(self, host_id: str) -> None: + host = self._call("GET", f"/host/{host_id}", missing_ok=True) + if not host: + return + if self._host_luns(host_id): + raise ArrayConflictError(f"Host {host_id} still has LUNs mapped") + view = self._by_name("/mappingview", _MV + host_id) + hg = self._by_name("/hostgroup", _HG + host_id) + lg = self._by_name("/lungroup", _LG + host_id) + if view is not None: + for obj_type, group in ((OBJ_HOSTGROUP, hg), (OBJ_LUNGROUP, lg)): + if group is not None: + self._call("PUT", "/mappingview/remove_associate", body={ + "ID": view["ID"], "ASSOCIATEOBJTYPE": obj_type, + "ASSOCIATEOBJID": group["ID"]}, tolerate=GROUP_NOT_IN_VIEW) + self._call("DELETE", f"/mappingview/{view['ID']}", missing_ok=True) + if hg is not None: + self._call("DELETE", "/host/associate", body={ + "ID": hg["ID"], "ASSOCIATEOBJTYPE": OBJ_HOST, "ASSOCIATEOBJID": host_id}, + tolerate=frozenset({HOST_NOT_IN_GROUP})) + self._call("DELETE", f"/hostgroup/{hg['ID']}", missing_ok=True) + if lg is not None: + self._call("DELETE", f"/lungroup/{lg['ID']}", missing_ok=True) + for collection, itype, action in ( + ("iscsi_initiator", TYPE_ISCSI_INITIATOR, "remove_iscsi_from_host"), + ("fc_initiator", TYPE_FC_INITIATOR, "remove_fc_from_host")): + # Cinder re-checks PARENTID on every row of this query; so do we, + # so that an ignored filter can never unbind another host's ports. + rows = self._list(f"/{collection}", {"PARENTTYPE": OBJ_HOST, + "PARENTID": host_id}) + for ini in rows: + if str(ini.get("PARENTID") or "") != host_id: + continue + self._call("PUT", f"/{collection}/{action}", + body={"ID": ini["ID"], "TYPE": itype}) + self._call("DELETE", f"/host/{host_id}", missing_ok=True) + + # -- mapping ------------------------------------------------------------------------------ + + def _host_luns(self, host_id: str) -> dict[str, int | None]: + """LUN id -> host LUN id for every LUN the host sees.""" + rows = self._list("/lun/associate", {"TYPE": OBJ_LUN, "ASSOCIATEOBJTYPE": OBJ_HOST, + "ASSOCIATEOBJID": host_id}) + out: dict[str, int | None] = {} + for r in rows: + lun_no: int | None = None + try: + meta = json.loads(r.get("ASSOCIATEMETADATA") or "{}") + if meta.get("HostLUNID") is not None: + lun_no = int(meta["HostLUNID"]) + except (TypeError, ValueError): + pass + out[str(r["ID"])] = lun_no + return out + + def _ensure_group(self, path: str, name: str, extra: dict[str, Any]) -> str: + found = self._by_name(path, name) + if found is None: + try: + found = self._call("POST", path, body={"NAME": name, **extra}) + except ArrayConflictError: # created concurrently + found = self._by_name(path, name) + if not found: + raise ArrayRequestError(f"Could not create {path[1:]} {name}") + return str(found["ID"]) + + def _ensure_view(self, host_id: str) -> str: + """Create (or complete) the host's group triple; returns the LUN group id.""" + if self._call("GET", f"/host/{host_id}", missing_ok=True) is None: + raise ArrayNotFoundError(f"Host {host_id} does not exist") + lg = self._ensure_group("/lungroup", _LG + host_id, {"APPTYPE": 0}) + hg = self._ensure_group("/hostgroup", _HG + host_id, {}) + groups = self._list("/hostgroup/associate", {"ASSOCIATEOBJTYPE": OBJ_HOST, + "ASSOCIATEOBJID": host_id}) + others = [g for g in groups if str(g.get("ID")) != hg] + if others: + raise ArrayConflictError( + f"Host {host_id} already belongs to host group " + f"{others[0].get('NAME') or others[0].get('ID')}; an OceanStor host can be " + "in one host group only") + if not groups: + self._call("POST", "/hostgroup/associate", body={ + "ID": hg, "ASSOCIATEOBJTYPE": OBJ_HOST, "ASSOCIATEOBJID": host_id}, + tolerate=frozenset({HOST_ALREADY_IN_GROUP})) + mv = self._ensure_group("/mappingview", _MV + host_id, {}) + for obj_type, group in ((OBJ_HOSTGROUP, hg), (OBJ_LUNGROUP, lg)): + self._call("PUT", "/mappingview/create_associate", body={ + "ID": mv, "ASSOCIATEOBJTYPE": obj_type, "ASSOCIATEOBJID": group}, + tolerate=GROUP_ALREADY_IN_VIEW) + return lg + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + seen = self._host_luns(host_id) + if volume_id in seen: + return Mapping(volume_id=volume_id, host_id=host_id, lun=seen[volume_id]) + lg = self._ensure_view(host_id) + self._call("POST", "/lungroup/associate", body={ + "ID": lg, "ASSOCIATEOBJTYPE": OBJ_LUN, "ASSOCIATEOBJID": volume_id}, + tolerate=frozenset({LUN_ALREADY_IN_GROUP, 1077948997})) + seen = self._host_luns(host_id) + if volume_id not in seen: + raise ArrayError(f"LUN {volume_id} not visible to host {host_id} after mapping") + return Mapping(volume_id=volume_id, host_id=host_id, lun=seen[volume_id]) + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + lg = self._by_name("/lungroup", _LG + host_id) + if lg is None: + return + self._call("DELETE", "/lungroup/associate", body={ + "ID": lg["ID"], "ASSOCIATEOBJTYPE": OBJ_LUN, "ASSOCIATEOBJID": volume_id}, + missing_ok=True) + + def list_mappings(self, volume_id: str) -> list[Mapping]: + groups = self._list("/lungroup/associate", {"ASSOCIATEOBJTYPE": OBJ_LUN, + "ASSOCIATEOBJID": volume_id}) + out = [] + for g in groups: + name = str(g.get("NAME", "")) + if not name.startswith(_LG): + continue + host_id = name[len(_LG):] + out.append(Mapping(volume_id=volume_id, host_id=host_id, + lun=self._host_luns(host_id).get(volume_id))) + return out diff --git a/simplyblock_core/storage_arrays/adapters/ibm_flashsystem.py b/simplyblock_core/storage_arrays/adapters/ibm_flashsystem.py new file mode 100644 index 0000000000..2e026294c9 --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/ibm_flashsystem.py @@ -0,0 +1,405 @@ +"""IBM FlashSystem / SAN Volume Controller (IBM Storage Virtualize) REST API. + +The Storage Virtualize REST API (port 7443) is a thin wrapper around the CLI: +every command is ``POST /rest/[/]`` with the command's +flags as a JSON object (``-unit b`` -> ``{"unit": "b"}``, ``-thin`` -> +``{"thin": true}``). Positional arguments go into the path. + +Auth: ``POST /rest/auth`` with headers ``X-Auth-Username`` / +``X-Auth-Password`` returns ``{"token": ...}``; every later call carries +``X-Auth-Token``. Tokens expire after an idle period (one hour by default) +and the array then answers 403, which the client turns into a re-login. + +Errors: a failed command answers HTTP 500 with the CLI's message, e.g. +``CMMVC5753E The specified object does not exist or is not a suitable +candidate.`` The ``CMMVCnnnnE`` code is mapped as follows (anything else is +an :class:`ArrayRequestError`): + +* not found: CMMVC5753E, CMMVC5754E, CMMVC5804E, CMMVC5842E +* conflict: CMMVC6035E (name exists), CMMVC6578E (iSCSI name already + assigned or invalid), CMMVC5867E (WWPN already assigned or invalid), + CMMVC9328E (NQN already assigned), CMMVC5878E (volume already mapped to + that host), CMMVC6071E (mapped to another host, needs ``force``), + CMMVC5840E (volume still mapped), CMMVC5871E (host still has mappings) + +Objects: volumes are addressed by their numeric id (``Volume.id``), hosts by +name (``ensure_host`` returns the host name). Note that the array reuses the +lowest free id after a deletion, so a stale volume id can address a +different volume later -- callers pass ``expected_identifier`` (the WWN) to +the client, which then refuses to act on the wrong volume. + +Volumes are created with ``mkvolume`` (``-thin`` unless ``thin`` is False); +the virtual capacity has 512-byte granularity (extents only govern the real +allocation). Resizing grows with ``expandvdisksize`` (a delta). Deletion uses +``rmvolume``. Volume identity: ``vdisk_UID`` (32 hex) -> ``naa.``. + +Hosts: ``mkhost`` takes exactly one of ``-fcwwpn`` / ``-iscsiname`` / +``-nqn``, so a host with FC and iSCSI initiators is created with one kind and +the other added with ``addhostport``. ``mkhost`` and ``addhostport`` always +carry ``-force`` (needed for FC WWPNs that are not logged in). + +Initiator ownership: ``mkhost``/``addhostport`` fail with CMMVC6578E / +CMMVC5867E / CMMVC9328E when a port already belongs to another host; the +adapter first reads its own host (``lshost ``, ports in the ``nodes`` +list) so ports it already owns are never re-added, and treats those codes as +:class:`ArrayConflictError`. + +Checked against vendor source +----------------------------- +IBM Ansible collection ``ibm.storage_virtualize`` (``module_utils/ +ibm_svc_utils.py``, ``modules/ibm_svc_host.py``, ``ibm_svc_manage_volume.py``, +``ibm_svc_vol_map.py``), IBM block CSI driver (``controllers/array_action/ +array_mediator_svc.py``) and the OpenStack Cinder Storwize driver +(``storwize_svc/storwize_const.py``, ``storwize_svc_common.py``): + +* REST root ``https://:7443/rest``, ``POST /rest/auth`` with + ``X-Auth-Username``/``X-Auth-Password`` -> ``{"token"}``, then + ``X-Auth-Token``; positional args in the path, flags as a JSON object. +* A failed command is HTTP 500 whose body is the CLI text with the + ``CMMVCnnnnE`` code (Ansible ``svc_obj_info`` reads the code from it). +* Codes: CMMVC5753E / CMMVC5754E / CMMVC5804E / CMMVC5842E (not found / + not mapped), CMMVC6035E (exists), CMMVC6071E (already mapped to a host), + CMMVC5867E / CMMVC6578E / CMMVC9328E (FC / iSCSI / NVMe port already + assigned or invalid). +* ``mkhost -protocol`` values ``scsi``, ``iscsi``, ``fcscsi``, ``tcpnvme``, + ``rdmanvme``, ``fcnvme``; ``mkhost``/``addhostport`` with ``-force``; + ``-fcwwpn`` ``:``-separated, ``-iscsiname``/``-nqn`` ``,``-separated, one + of them per ``mkhost``. +* ``lshost `` ports under ``nodes`` (``WWPN``, ``iscsi_name``, + ``nqn``); ``lsvdisk `` is a list (volume, then copies); + ``lsvdiskhostmap`` with one row may be a bare dict (``host_name``, + ``SCSI_id``); ``mkvdiskhostmap -force``. +* ``mkvolume`` with ``size``/``unit=b``/``thin``/``iogrp``, + ``expandvdisksize`` with a byte delta, ``rmvolume``; ``lssystem`` + ``code_level`` is ``" (build ...)"``. + +Verify against a real system +---------------------------- +* ``/rest/v1`` on code levels that reject ``/rest`` (no vendor source uses + it); set the ``rest_root`` option. +* HTTP status on an expired token (assumed 403, 401 handled the same) and on + bad credentials at ``/rest/auth`` (assumed 403); the Ansible collection + only special-cases 429. +* Codes CMMVC5840E (rmvolume of a mapped volume), CMMVC5871E (rmhost with + mappings) and CMMVC5878E (volume already mapped to that host) are from + memory of the CLI reference; no vendor driver handles them (Cinder and CSI + check ``lsvdiskhostmap`` first, as this adapter does). +* NVMe ``-protocol`` on levels before 8.6 (``nvme`` only?) -- set + ``nvme_protocol_value`` if the array rejects ``tcpnvme`` / ``rdmanvme``. +* Whether a SCSI host may hold both FC and iSCSI ports (``addhostport`` of the + other kind) on code levels that split ``-protocol`` into ``fcscsi`` and + ``iscsi``. SCSI hosts are created without ``-protocol`` (default ``scsi``). +""" +from __future__ import annotations + +import re +from typing import Any +from urllib.parse import quote + +from ..base import ( + ArrayAdapter, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, +) +from ..http import body_text, json_of + +_CODE_RE = re.compile(r"CMMVC\d{4}[EW]") + +NOT_FOUND_CODES = frozenset({"CMMVC5753E", "CMMVC5754E", "CMMVC5804E", "CMMVC5842E"}) +CONFLICT_CODES = frozenset({ + "CMMVC6035E", # object already exists + "CMMVC6578E", # iSCSI name already assigned or not valid + "CMMVC5867E", # WWPN already assigned or not valid + "CMMVC9328E", # NQN already assigned + "CMMVC5878E", # volume already mapped to this host + "CMMVC6071E", # volume mapped to another host; needs -force + "CMMVC5840E", # volume not deleted: still mapped + "CMMVC5871E", # host has mappings +}) +#: Codes meaning "this port is owned by another host" on mkhost/addhostport. +PORT_TAKEN_CODES = frozenset({"CMMVC6578E", "CMMVC5867E", "CMMVC9328E"}) +_PORT_SEP = {"nqn": ",", "fcwwpn": ":", "iscsiname": ","} + +_NVME_PROTOCOL = {Protocol.NVME_TCP: "tcpnvme", Protocol.NVME_RDMA: "rdmanvme"} + + +def ibm_wwn(vdisk_uid: str) -> str: + return "naa." + vdisk_uid.lower() + + +def _rows(data: Any) -> list[dict[str, Any]]: + """List views can come back as a bare dict when there is one row.""" + if data is None: + return [] + if isinstance(data, dict): + return [data] + return [d for d in data if isinstance(d, dict)] + + +def _detail(data: Any) -> dict[str, Any]: + """Detail views: a dict, or a list (object first, then e.g. copies).""" + rows = _rows(data) + return rows[0] if rows else {} + + +class IbmFlashSystemAdapter(ArrayAdapter): + vendor = "ibm-flashsystem" + display_name = "IBM FlashSystem / Storage Virtualize" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC, Protocol.NVME_TCP, + Protocol.NVME_RDMA}), + size_granularity_bytes=512, + min_size_bytes=1 << 20, + max_name_len=63, + ) + credential_fields = (frozenset({"username", "password"}),) + option_docs = ( + OptionDoc("pool", "Storage pool (mdisk group) name or id for new volumes.", + required=True), + OptionDoc("thin", "Create thin-provisioned volumes (mkvolume -thin). In a " + "data reduction pool this is a DRP thin volume.", default=True), + OptionDoc("io_group", "I/O group (name or id) that owns new volumes; " + "default: chosen by the array."), + OptionDoc("rest_root", "REST path root; '/rest/v1' on code levels that " + "reject '/rest'.", default="/rest"), + OptionDoc("nvme_protocol_value", "Override the mkhost -protocol value used " + "for NVMe hosts (default: tcpnvme / rdmanvme by transport)."), + ) + + def __init__(self, ctx): + super().__init__(ctx) + self._root = "/" + str(self.option("rest_root")).strip("/") + + # -- session -------------------------------------------------------------- + + def login(self) -> None: + self.session.headers.pop("X-Auth-Token", None) + resp = self.session.post(f"{self._root}/auth", headers={ + "X-Auth-Username": self.secret("username"), + "X-Auth-Password": self.secret("password"), + }) + token = (json_of(resp) or {}).get("token") + if not token: + raise ArrayRequestError("Storage Virtualize /auth returned no token") + self.session.headers["X-Auth-Token"] = token + + def close(self) -> None: + # There is no logout command; tokens lapse on their own. + self.session.headers.pop("X-Auth-Token", None) + + # -- request helper --------------------------------------------------------- + + def _cmd(self, command: str, obj: str | None = None, *, missing_ok: bool = False, + **opts: Any) -> Any: + """Run one CLI command. Returns the decoded JSON (``None`` on an empty + body, or when ``missing_ok`` and the object does not exist).""" + path = f"{self._root}/{command}" + if obj is not None: + path += "/" + quote(str(obj), safe="") + resp = self.session.post(path, json=opts, ok=(500,)) + if resp.status_code == 500: + text = body_text(resp) + m = _CODE_RE.search(text) + code = m.group(0) if m else "" + msg = f"{command}{' ' + str(obj) if obj is not None else ''}: {text.strip()}" + if code in NOT_FOUND_CODES: + if missing_ok: + return None + raise ArrayNotFoundError(msg) + if code in CONFLICT_CODES: + raise ArrayConflictError(msg) + raise ArrayRequestError(msg, status=500) + if not resp.content: + return None + try: + return resp.json() + except ValueError as e: + raise ArrayRequestError(f"{command}: expected JSON, got {body_text(resp)}") from e + + # -- volumes ---------------------------------------------------------------- + + @staticmethod + def _volume(row: dict[str, Any]) -> Volume: + uid = str(row.get("vdisk_UID") or "") + return Volume(id=str(row["id"]), name=row["name"], size_bytes=int(row["capacity"]), + wwn=ibm_wwn(uid) if uid else None) + + def get_info(self) -> ArrayInfo: + s = _detail(self._cmd("lssystem")) + return ArrayInfo(vendor=self.vendor, model=str(s.get("product_name", "")), + version=str(s.get("code_level", "")).split(" ")[0], + serial=str(s.get("id", ""))) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + opts: dict[str, Any] = {"name": name, "pool": str(self.option("pool")), + "size": size_bytes, "unit": "b"} + if self.option("thin"): + opts["thin"] = True + if self.option("io_group") not in (None, ""): + opts["iogrp"] = str(self.option("io_group")) + out = self._cmd("mkvolume", **opts) or {} + vid = out.get("id") + vol = self.get_volume(str(vid)) if vid is not None else self.find_volume(name) + if vol is None: + raise ArrayRequestError(f"mkvolume {name} reported success but the volume " + "cannot be read back") + return vol + + def get_volume(self, volume_id: str) -> Volume | None: + row = _detail(self._cmd("lsvdisk", volume_id, missing_ok=True, bytes=True)) + return self._volume(row) if row else None + + def find_volume(self, name: str) -> Volume | None: + rows = _rows(self._cmd("lsvdisk", filtervalue=f"name={name}", bytes=True)) + rows = [r for r in rows if r.get("name") == name] + return self._volume(rows[0]) if rows else None + + def _require(self, volume_id: str) -> Volume: + vol = self.get_volume(volume_id) + if vol is None: + raise ArrayNotFoundError(f"Volume {volume_id} does not exist") + return vol + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + vol = self._require(volume_id) + delta = size_bytes - vol.size_bytes + if delta < 0: + raise ArrayConflictError( + f"Refusing to shrink volume {vol.name} ({vol.size_bytes} -> {size_bytes})") + if delta: + self._cmd("expandvdisksize", vol.id, size=delta, unit="b") + return self._require(vol.id) + + def delete_volume(self, volume_id: str) -> None: + self._cmd("rmvolume", volume_id) + + # -- hosts ------------------------------------------------------------------- + + def _host(self, name: str) -> dict[str, Any] | None: + data = self._cmd("lshost", name, missing_ok=True) + if data is None: + return None + merged: dict[str, Any] = {} + nodes: list[dict[str, Any]] = [] + for row in _rows(data): + nodes.extend(n for n in row.get("nodes", []) if isinstance(n, dict)) + merged.update({k: v for k, v in row.items() if k != "nodes"}) + merged["nodes"] = nodes + return merged + + @staticmethod + def _ports(host: dict[str, Any]) -> dict[str, set[str]]: + ports: dict[str, set[str]] = {"iscsiname": set(), "fcwwpn": set(), "nqn": set()} + for n in host.get("nodes", []): + if n.get("WWPN"): + ports["fcwwpn"].add(str(n["WWPN"]).lower()) + if n.get("iscsi_name"): + ports["iscsiname"].add(str(n["iscsi_name"]).lower()) + if n.get("nqn"): + ports["nqn"].add(str(n["nqn"])) + return ports + + def _nvme_protocol(self, spec: HostSpec) -> str | None: + nvme = {p for p in spec.protocols if p.is_nvme} + if not nvme: + return None + if len(nvme) > 1: + raise ArrayUnsupportedError( + "A Storage Virtualize NVMe host has one transport; register " + f"{', '.join(sorted(p.value for p in nvme))} initiators as separate hosts") + override = self.option("nvme_protocol_value") + return str(override) if override else _NVME_PROTOCOL[next(iter(nvme))] + + def ensure_host(self, spec: HostSpec) -> str: + wanted = { + "nqn": spec.addresses(Protocol.NVME_TCP, Protocol.NVME_RDMA), + "fcwwpn": [a.upper() for a in spec.addresses(Protocol.FC)], + "iscsiname": spec.addresses(Protocol.ISCSI), + } + protocol = self._nvme_protocol(spec) + host = self._host(spec.name) + try: + if host is None: + # mkhost takes exactly one of -fcwwpn / -iscsiname / -nqn; the + # other port kinds are added with addhostport below. + kind = next(k for k, v in wanted.items() if v) + opts: dict[str, Any] = {"name": spec.name, + kind: _PORT_SEP[kind].join(wanted[kind]), + "force": True} + if protocol: + opts["protocol"] = protocol + self._cmd("mkhost", **opts) + have = {k: {v.lower() for v in wanted[kind]} if k == kind else set() + for k in wanted} + else: + have = self._ports(host) + for key, values in wanted.items(): + missing = [v for v in values if v.lower() not in have[key] + and v not in have[key]] + if missing: + port_opts: dict[str, Any] = {key: _PORT_SEP[key].join(missing), + "force": True} + self._cmd("addhostport", spec.name, **port_opts) + except ArrayConflictError as e: + code = _CODE_RE.search(str(e)) + if code and code.group(0) in PORT_TAKEN_CODES: + raise ArrayConflictError( + f"An initiator of host {spec.name} is registered to another host " + f"(or is invalid): {e}") from e + raise + return spec.name + + def delete_host(self, host_id: str) -> None: + try: + self._cmd("rmhost", host_id) + except ArrayNotFoundError: + pass + + # -- mappings ------------------------------------------------------------------- + + def _maps(self, volume_id: str) -> list[dict[str, Any]]: + data = self._cmd("lsvdiskhostmap", volume_id, missing_ok=True) + if data is None: + raise ArrayNotFoundError(f"Volume {volume_id} does not exist") + return _rows(data) + + @staticmethod + def _mapping(volume_id: str, row: dict[str, Any]) -> Mapping: + scsi = str(row.get("SCSI_id") or "") + return Mapping(volume_id=volume_id, host_id=str(row["host_name"]), + lun=int(scsi) if scsi.isdigit() else None) + + def list_mappings(self, volume_id: str) -> list[Mapping]: + return [self._mapping(volume_id, r) for r in self._maps(volume_id) + if r.get("host_name")] + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + for m in self.list_mappings(volume_id): + if m.host_id == host_id: + return m + try: + # force: allow the volume to be mapped to more than one host. + self._cmd("mkvdiskhostmap", volume_id, host=host_id, force=True) + except ArrayConflictError as e: + if "CMMVC5878E" not in str(e): # raced with another mapper: fine + raise + for m in self.list_mappings(volume_id): + if m.host_id == host_id: + return m + raise ArrayError(f"Mapping of volume {volume_id} to host {host_id} not found " + "after mkvdiskhostmap") + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + if not any(m.host_id == host_id for m in self.list_mappings(volume_id)): + return + self._cmd("rmvdiskhostmap", volume_id, missing_ok=True, host=host_id) diff --git a/simplyblock_core/storage_arrays/adapters/infinidat.py b/simplyblock_core/storage_arrays/adapters/infinidat.py new file mode 100644 index 0000000000..d1bdef5394 --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/infinidat.py @@ -0,0 +1,329 @@ +"""Infinidat InfiniBox, REST API ``/api/rest`` (InfiniBox 7.x). + +Auth: HTTP basic on every request (``username`` / ``password`` credential). +The array also sets a session cookie, which ``requests`` keeps; basic auth +is still sent, so an expired cookie never matters and a rotated password +surfaces as a 401 (``ArrayClient`` then re-resolves the credential). + +Every response is an envelope ``{"result": ..., "error": ..., "metadata": +{...}}``; errors carry ``error.code`` (e.g. ``VOLUME_NOT_FOUND``) and +``error.message``. Lists are paged (``page``, ``page_size`` up to 1000; +``metadata.pages_total``) and filtered with ``field=eq:value``. + +Operations the array considers risky need ``?approved=true``; without it +the array answers ``APPROVAL_REQUIRED`` (or a more specific +``APPROVAL_REQUIRED_*`` code such as ``..._VOLUME_HAS_CHILDREN``). Like +infinisdk in non-interactive mode, the adapter sends the flag on *every* +non-GET request, and maps any ``APPROVAL_REQUIRED*`` code to +:class:`ArrayRequestError` (never to an auth failure, even on 403). + +Objects have integer ids, returned here as strings. Volumes are created in +the pool named by the ``pool`` option (resolved to its id once per session), +thin by default. Sizes are bytes; see granularity below. + +Volume identity: ``serial`` is the SCSI NAA-6 identifier without the +leading ``6`` (``742b0f...``, 31 hex digits, IEEE company id 0x742B0F); +``wwn = "naa.6" + serial``. A 32-digit serial already starting with ``6`` is +used as-is. + +Hosts: ``POST /hosts {name}`` then one ``POST /hosts/{id}/ports +{type, address}`` per initiator (``ISCSI`` IQN; ``FC`` WWPN as 16 lowercase +hex digits). A port owned by another host is refused by the array; when that +happens on a host ``ensure_host`` just created, the empty host is deleted +again. Mappings: ``POST /hosts/{id}/luns {volume_id}`` -> ``{lun, ...}``; +``GET /volumes/{id}/luns`` lists them; ``DELETE +/hosts/{id}/luns/volume_id/{vid}?approved=true`` unmaps. Mappings made +through a host *cluster* (``clustered: true``) are not listed by this adapter +(it never creates them), so ``delete_volume`` on such a volume fails. + +Simulator: infinisdk's test extra pins ``infinisim`` (and its doctests run +against ``infinisim.infinibox.Infinibox``), but infinisim is Infinidat's +internal package: it is on neither PyPI nor GitHub, so no public simulator +can back a live contract run. + +Checked against vendor source +----------------------------- +(Infinidat ``infinisdk``: ``core/api/api.py``, ``core/config.py``, +``core/field_filter.py``, ``core/object_query.py``, +``core/translators_and_types.py``, ``infinibox/scsi_serial.py``, +``infinibox/dataset.py``, ``infinibox/system_object.py``; Infinidat +``infinibox-csi-driver``: ``iboxapi/host.go``, ``volumes.go``, +``system.go``, ``storage/common/*.go``; Infinidat ``api_7_3`` Postman +collection.) + +* Envelope ``{result, error, metadata}``; ``metadata`` has ``page``, + ``page_size``, ``pages_total``, ``number_of_objects``; ``page_size=1000`` + works (infinisdk and the CSI driver use it); filters ``field=eq:value``. +* Basic auth on every request; an expired session answers 401 (infinisdk + logs in again on 401). +* Approval: codes ``APPROVAL_REQUIRED`` and + ``APPROVAL_REQUIRED_VOLUME_HAS_CHILDREN``; infinisdk adds + ``approved=true`` to every non-GET request. +* Not found is HTTP 404 (infinisdk ``is_in_system``) with codes + ``VOLUME_NOT_FOUND``, ``HOST_NOT_FOUND``, ``POOL_NOT_FOUND``, + ``LUN_NOT_FOUND``; mapping twice answers ``MAPPING_ALREADY_EXISTS``. +* ``serial``: infinisdk parses it as IEEE company id (first 6 hex digits, + ``742b0f``) + reserved + system id + 64-bit volume id, i.e. *without* the + NAA type nibble; hosts see ``naa.6742b0f...`` (CSI driver). +* ``PUT /volumes/{id} {"size"}`` sets the absolute size (infinisdk + ``resize`` sends current + delta); shrinking needs ``force_resize``. +* Ports: ``{type, address}`` with type ``FC``/``ISCSI``/``NVME`` + (case-insensitive: infinisdk sends ``fc``/``iscsi``, the CSI driver + ``FC``/``ISCSI``/``NVME``); FC WWPNs are accepted with colons (infinisdk) + and as 16 bare hex digits (CSI driver). +* ``POST /hosts/{id}/luns {volume_id[, lun]}`` -> LUN object; + ``GET /volumes/{id}/luns`` entries carry ``lun``, ``host_id``, + ``host_cluster_id``, ``clustered``, ``volume_id``. +* ``GET /system``: ``model``, ``full_model``, ``serial_number`` (an + integer), ``version``, ``name``. + +Verify against a real system +---------------------------- +* Size granularity/minimum: the adapter rounds to 1 MiB (minimum 1 MiB). + Volumes have ``num_blocks`` (512-byte blocks), and the CSI driver only + ever requests whole GB/GiB with a 1 GB minimum; if the array rounds MiB + sizes up, raise ``size_granularity_bytes`` (a silent round-up would make + later resizes look like shrinks). +* Codes/statuses the fake invents: name conflicts (``*_NAME_CONFLICT``, + 409?), a port owned by another host (``PORT_ALREADY_BELONGS_TO_HOST``, + 409?), deleting a mapped volume or a host with LUNs, and the HTTP status + of ``APPROVAL_REQUIRED`` (403?). +* The FC address format the array *returns* (normalised on read either way). +* That auto-assigned LUN numbers start at 1. +* Name limit (adapter declares 63 characters). +* NVMe/TCP (port type ``NVME``, host NQN) is not implemented. +""" +from __future__ import annotations + +from typing import Any + +from ..base import ( + ArrayAdapter, + ArrayAuthError, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, + normalize_wwpn, +) +from ..http import body_text + +_API = "/api/rest" +_PAGE_SIZE = 1000 +_APPROVED = {"approved": "true"} + + +def infinibox_wwn(serial: str) -> str: + s = serial.strip().lower() + if len(s) == 32 and s.startswith("6"): + return "naa." + s + return "naa.6" + s + + +def _port_key(ptype: str, address: str) -> tuple[str, str]: + ptype = ptype.upper() + if ptype == "FC": + try: + return ptype, normalize_wwpn(address) + except ValueError: + pass + return ptype, address.strip().lower() + + +class InfinidatAdapter(ArrayAdapter): + vendor = "infinidat-infinibox" + display_name = "Infinidat InfiniBox" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC}), + size_granularity_bytes=1 << 20, + min_size_bytes=1 << 20, + max_name_len=63, + ) + option_docs = ( + OptionDoc("pool", "Name of the pool new volumes are created in.", required=True), + OptionDoc("provtype", "Provisioning type of new volumes: THIN or THICK.", + default="THIN"), + ) + + def __init__(self, ctx): + super().__init__(ctx) + if str(self.option("provtype")).upper() not in ("THIN", "THICK"): + raise ArrayError("InfiniBox option provtype must be THIN or THICK") + self._pool_id: int | None = None + + # -- session ------------------------------------------------------------ + + def login(self) -> None: + self.session.http.cookies.clear() + self.session.set_basic_auth(self.secret("username"), self.secret("password")) + self._pool_id = None + + # -- request helpers ----------------------------------------------------- + + def _send(self, method: str, path: str, *, params: dict[str, Any] | None = None, + json: Any = None, missing_ok: bool = False) -> dict[str, Any] | None: + """Return the response envelope, or ``None`` for a missing object when + ``missing_ok``; map envelope errors to :class:`ArrayError` types. + Every non-GET request is pre-approved (``approved=true``), as infinisdk + does.""" + if method.upper() != "GET": + params = {**(params or {}), **_APPROVED} + resp = self.session.request(method, _API + path, params=params, json=json, + ok=(400, 403, 404, 409, 500)) + try: + body = resp.json() if resp.content else None + except ValueError: + body = None + env: dict[str, Any] = body if isinstance(body, dict) else {} + err = env.get("error") + status = resp.status_code + if status < 300 and not err: + return env + err = err if isinstance(err, dict) else {} + code = str(err.get("code") or "") + msg = str(err.get("message") or "") or body_text(resp) + text = f"{method} {path} -> HTTP {status}: {code} {msg}".strip() + if code.startswith("APPROVAL_REQUIRED"): + raise ArrayRequestError(text, status=status) + if status == 403: + raise ArrayAuthError(text) + if status == 404 or code.endswith("NOT_FOUND"): + if missing_ok: + return None + raise ArrayNotFoundError(text) + if status == 409 or "CONFLICT" in code or "ALREADY" in code: + raise ArrayConflictError(text) + raise ArrayRequestError(text, status=status) + + def _call(self, method: str, path: str, *, params: dict[str, Any] | None = None, + json: Any = None, missing_ok: bool = False) -> Any: + env = self._send(method, path, params=params, json=json, missing_ok=missing_ok) + return None if env is None else env.get("result") + + def _list(self, path: str, **filters: str) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + page = 1 + while True: + params: dict[str, Any] = {k: f"eq:{v}" for k, v in filters.items()} + params.update(page=page, page_size=_PAGE_SIZE) + env = self._send("GET", path, params=params) or {} + out.extend(env.get("result") or []) + pages = int((env.get("metadata") or {}).get("pages_total") or 1) + if page >= pages: + return out + page += 1 + + @staticmethod + def _volume(v: dict[str, Any]) -> Volume: + serial = str(v.get("serial") or "") + return Volume(id=str(v["id"]), name=v["name"], size_bytes=int(v["size"]), + wwn=infinibox_wwn(serial) if serial else None) + + @staticmethod + def _int_id(object_id: str) -> int | None: + try: + return int(object_id) + except (TypeError, ValueError): + return None + + def _pool(self) -> int: + if self._pool_id is None: + name = self.option("pool") + pools = self._list("/pools", name=name) + if not pools: + raise ArrayNotFoundError(f"InfiniBox pool {name!r} not found") + self._pool_id = int(pools[0]["id"]) + return self._pool_id + + # -- contract ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + system = self._call("GET", "/system") or {} + model = system.get("model") or system.get("full_model") or "InfiniBox" + return ArrayInfo(vendor=self.vendor, model=str(model), + version=str(system.get("version") or ""), + serial=str(system.get("serial_number") or "")) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + result = self._call("POST", "/volumes", json={ + "name": name, "size": size_bytes, "pool_id": self._pool(), + "provtype": str(self.option("provtype")).upper()}) + return self._volume(result) + + def get_volume(self, volume_id: str) -> Volume | None: + if self._int_id(volume_id) is None: + return None + result = self._call("GET", f"/volumes/{volume_id}", missing_ok=True) + return self._volume(result) if result else None + + def find_volume(self, name: str) -> Volume | None: + items = self._list("/volumes", name=name) + return self._volume(items[0]) if items else None + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + result = self._call("PUT", f"/volumes/{volume_id}", json={"size": size_bytes}) + return self._volume(result) + + def delete_volume(self, volume_id: str) -> None: + self._call("DELETE", f"/volumes/{volume_id}", params=_APPROVED) + + # -- hosts ---------------------------------------------------------------- + + def ensure_host(self, spec: HostSpec) -> str: + wanted = ([("ISCSI", a) for a in spec.addresses(Protocol.ISCSI)] + + [("FC", a) for a in spec.addresses(Protocol.FC)]) + hosts = self._list("/hosts", name=spec.name) + created = not hosts + host = hosts[0] if hosts else self._call("POST", "/hosts", json={"name": spec.name}) + host_id = int(host["id"]) + have = {_port_key(str(p.get("type", "")), str(p.get("address", ""))) + for p in host.get("ports") or []} + try: + for ptype, address in wanted: + if _port_key(ptype, address) in have: + continue + self._call("POST", f"/hosts/{host_id}/ports", + json={"type": ptype, "address": address}) + except ArrayError: + if created: + try: + self._call("DELETE", f"/hosts/{host_id}", params=_APPROVED, + missing_ok=True) + except ArrayError: + pass + raise + return str(host_id) + + def delete_host(self, host_id: str) -> None: + if self._int_id(host_id) is None: + return + self._call("DELETE", f"/hosts/{host_id}", params=_APPROVED, missing_ok=True) + + # -- mappings --------------------------------------------------------------- + + def list_mappings(self, volume_id: str) -> list[Mapping]: + luns = self._call("GET", f"/volumes/{volume_id}/luns") or [] + return [Mapping(volume_id=str(volume_id), host_id=str(m["host_id"]), lun=int(m["lun"])) + for m in luns if m.get("host_id") and not m.get("clustered")] + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + for m in self.list_mappings(volume_id): + if m.host_id == host_id: + return m + result = self._call("POST", f"/hosts/{host_id}/luns", + json={"volume_id": self._int_id(volume_id)}) + return Mapping(volume_id=str(volume_id), host_id=str(host_id), lun=int(result["lun"])) + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + luns = self._call("GET", f"/volumes/{volume_id}/luns", missing_ok=True) or [] + if not any(str(m.get("host_id")) == str(host_id) for m in luns): + return + self._call("DELETE", f"/hosts/{host_id}/luns/volume_id/{volume_id}", + params=_APPROVED, missing_ok=True) diff --git a/simplyblock_core/storage_arrays/adapters/netapp_ontap.py b/simplyblock_core/storage_arrays/adapters/netapp_ontap.py new file mode 100644 index 0000000000..4b0ece5cea --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/netapp_ontap.py @@ -0,0 +1,876 @@ +"""NetApp ONTAP 9.x REST API (AFF/FAS/ASA; also Lenovo ThinkSystem DM). + +Auth: HTTP basic (``username``/``password``) on every request; there is no +session to establish. ``login`` checks that the configured SVM exists. + +Layout (``volume_per_lun``, the default): every simplyblock volume is its +own FlexVol ``sb_`` (FlexVol names cannot contain ``-``) holding one +LUN or NVMe namespace ``sb-``. The FlexVol is thin (``guarantee none``), +has no snapshot reserve and no snapshot policy, and is sized +``flexvol_headroom_percent`` above the LUN so a fully written thin LUN does +not run its volume out of space. With ``volume_per_lun: false`` LUNs are +placed in one existing FlexVol (``flexvol``) that the operator sizes. + +Protocol family. ``create_volume`` has no protocol argument, so the option +``protocol_family`` (``scsi`` default, or ``nvme``) decides whether new +volumes are LUNs (``/api/storage/luns``) or namespaces +(``/api/storage/namespaces``). Existing volumes of either kind are always +found, mapped and deleted correctly. ``capabilities.protocols`` is a class +attribute, so it lists every protocol ONTAP serves; mapping a LUN to an NVMe +host (or a namespace to a SCSI host) raises :class:`ArrayUnsupportedError`. + +SCSI hosts are igroups (``protocol mixed``, ``os_type linux``); the host id +is the igroup name; volumes are mapped with ``lun-maps`` and the mapping's +``lun`` is the ``logical_unit_number``. Volume identity: ``naa.600a0980`` + +hex of the 12-character ASCII LUN serial number. + +NVMe hosts. ONTAP maps a namespace to exactly one subsystem, so a subsystem +cannot stand for a host if a volume is to be attached to several hosts. +Instead: + +* ``ensure_host`` creates a *host subsystem* named after the host holding + the host NQNs (and nothing else). The host id is its name. +* ``map_volume`` maps the namespace to its own *volume subsystem* + (``sb_``, comment :data:`VOLUME_SUBSYSTEM_COMMENT`, created on first + map) and adds the host's NQNs to it; ``unmap_volume`` removes them. +* The mapping's ``lun`` is the namespace's NSID in the volume subsystem. + +Volume subsystems are recognised by their comment and never count as hosts. +Namespaces carry no serial-derived identity in the REST API, so their +:attr:`Volume.identifier` is ``id.`` (the same UUID the host +sees as ``/dev/disk/by-id/nvme-uuid.``); ``nguid`` is left empty. + +ONTAP lets an initiator / host NQN sit in several igroups / subsystems. The +contract does not: before adding one, the adapter looks it up and raises +:class:`ArrayConflictError` if another host (igroup or host subsystem) has it. + +Asynchronous calls answer ``202`` with a job link: FlexVol create/resize/ +delete always, LUN and namespace create/modify/delete on some systems. The +adapter polls ``/api/cluster/jobs/{uuid}`` until ``success`` or +``failure``, at most ``job_timeout_s`` seconds. + +A deleted FlexVol goes to ONTAP's volume recovery queue (still using space) +unless ``skip_recovery_queue`` is set, which sends ``force=true``. + +An igroup's ``protocol`` cannot be changed after creation, so an existing +``iscsi``/``fcp`` igroup (not created by this adapter) that would need the +other kind of initiator raises :class:`ArrayConflictError`. + +ASA r2 (``disaggregated`` clusters, ONTAP 9.16.1+) has no FlexVols; ``login`` +detects it and raises :class:`ArrayUnsupportedError`. + +Checked against vendor source +----------------------------- +Trident = github.com/NetApp/trident ``storage_drivers/ontap/api`` (its REST +client under ``rest/`` is generated from the ONTAP OpenAPI spec); Ansible = +``netapp.ontap`` collection; Cinder = ``volume/drivers/netapp/dataontap``. + +* FlexVol ``DELETE`` of an online volume needs no offline first (Ansible + ``na_ontap_volume.py`` ``rest_delete_volume``; Trident ``ontap_rest.go`` + ``destroyVolumeByUUID``); ``force`` bypasses the recovery queue + (``rest/client/storage/volume_delete_parameters.go``). +* ``aggregates`` is optional on FlexVol ``POST`` (``rest/models/volume.go``; + Ansible ``aggregates_rest`` omits it when no aggregate is given); ``|`` + is the OR operator in queries (Trident ``ontap_rest.go`` aggregates query). +* LUN/namespace ``POST``/``PATCH``/``DELETE`` and FlexVol + ``POST``/``PATCH``/``DELETE`` may answer ``202`` + job; igroup, lun-map, + subsystem, subsystem-map and subsystem-host calls are synchronous + (``rest/client/*/*_responses.go``; Trident ``lunCreate``). +* Error codes: ``4`` entry doesn't exist, ``1`` duplicate entry, ``5374242`` + LUN exists, ``5702832`` LUN create in progress, ``72090025`` subsystem + exists, ``5374922`` LUN map exists; a missing object addressed by UUID is + HTTP 404 (Trident ``abstraction_error.go``, ``ontap_rest.go`` + ``isRestNotFoundError``). The adapter classifies by status, code and + message text. +* igroup ``protocol`` is ``fcp``/``iscsi``/``mixed`` (default ``mixed``) and + "cannot be changed after creation of the group" (``rest/models/igroup.go``). +* ``nsid`` is 8 hex digits plus ``h`` (``"00000001h"``, + ``rest/models/nvme_subsystem_map.go``); paths + ``/protocols/nvme/subsystem-maps/{subsystem.uuid}/{namespace.uuid}``, + ``/protocols/nvme/subsystems/{subsystem.uuid}/hosts/{nqn}``, + ``/protocols/san/lun-maps/{lun.uuid}/{igroup.uuid}``, + ``/protocols/san/igroups/{igroup.uuid}/initiators`` + (``rest/client/n_v_me/nv_me_client.go``, ``s_a_n`` client). +* ``allow_delete_with_hosts`` on subsystem ``DELETE`` (Trident + ``NVMeSubsystemDelete``); namespace resize via ``PATCH space.size`` + (Trident ``NVMeNamespaceSetSize``). +* ``records`` arrays for bulk igroup-initiator and subsystem-host ``POST`` + (``rest/models/igroup_initiator.go``, ``nvme_subsystem_host.go``). +* NVMe hosts (NVMe/FC included) are subsystem ``hosts[].nqn`` only + (``rest/models/nvme_subsystem_host.go``). +* ``naa.600a0980`` + hex of the 12-character LUN serial (Ansible + ``na_ontap_lun_map.py``, ``na_ontap_rest_info.py``); ``serial_number`` + is exactly 12 characters (``rest/models/lun.go``). +* ASA r2 detection: ``GET /cluster?fields=disaggregated`` on 9.16.1+ + (Trident ``ontap_rest.go`` ``MinimumASAR2Version``; Cinder + ``client_cmode_rest_asar2.py`` uses ``/storage/storage-units``). + +Verify against a real system +---------------------------- +* A thin LUN/namespace may not be larger than its FlexVol (the fake + enforces this; it is why resize grows the FlexVol first). +* With several ``aggregates`` configured the adapter picks the one with the + most ``space.block_storage.available``; needs read access to + ``/api/storage/aggregates``, which SVM-scoped accounts may lack. +* Whether a duplicate FlexVol name fails synchronously (``400``/``409``) or + as a failed job, and its exact message text. +* Whether query values match case-insensitively (WWPNs are always written in + lowercase colon form, IQNs in lowercase). +""" +from __future__ import annotations + +from collections.abc import Callable +from typing import Any +from urllib.parse import quote + +import requests + +from ..base import ( + ArrayAdapter, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + Capabilities, + HostSpec, + Initiator, + Mapping, + OptionDoc, + Protocol, + Volume, + colon_wwpn, +) +from ..http import json_of + +NETAPP_NAA_PREFIX = "naa.600a0980" +VOLUME_SUBSYSTEM_COMMENT = "simplyblock volume subsystem" +FLEXVOL_COMMENT = "simplyblock" +MIN_FLEXVOL_BYTES = 20 << 20 +_FAMILIES = ("scsi", "nvme") +_GUARANTEES = ("none", "volume") +_LUN = "lun" +_NS = "namespace" + +#: ONTAP error codes (Trident ``storage_drivers/ontap/api/abstraction_error.go``). +_NOT_FOUND_CODES = frozenset({"4"}) +_CONFLICT_CODES = frozenset({"1", "5374242", "5702832", "72090025", "5374922"}) +_ASA_R2_MIN_VERSION = (9, 16, 1) + +_LUN_FIELDS = "uuid,name,serial_number,space.size,location.logical_unit,location.volume,svm" +_NS_FIELDS = "uuid,name,space.size,location.namespace,location.volume,svm" + + +def ontap_wwn(serial: str) -> str: + """``naa.600a0980`` + hex of the 12-character ASCII LUN serial.""" + return NETAPP_NAA_PREFIX + serial.encode("ascii").hex() + + +def flexvol_name(name: str) -> str: + """FlexVol names allow letters, digits and ``_`` only.""" + return name.replace("-", "_") + + +def parse_nsid(value: Any) -> int | None: + """``"00000001h"`` (ONTAP's NSID format) or an int -> ``1``.""" + if value is None: + return None + if isinstance(value, int): + return value + text = str(value).strip().lower().removesuffix("h") + try: + return int(text, 16) + except ValueError: + return None + + +def _as_bool(value: Any) -> bool: + if isinstance(value, str): + return value.strip().lower() in ("1", "true", "yes", "on") + return bool(value) + + +class NetAppOntapAdapter(ArrayAdapter): + vendor = "netapp-ontap" + display_name = "NetApp ONTAP" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC, Protocol.NVME_TCP, Protocol.NVME_FC}), + size_granularity_bytes=4096, + min_size_bytes=1 << 20, + max_name_len=96, + ) + credential_fields = (frozenset({"username", "password"}),) + option_docs = ( + OptionDoc("svm", "Storage VM (vserver) that owns the volumes, igroups and " + "subsystems.", required=True), + OptionDoc("protocol_family", "'scsi' (LUNs, for iSCSI/FC hosts) or 'nvme' " + "(namespaces, for NVMe/TCP and NVMe/FC hosts): what new volumes " + "are created as.", default="scsi"), + OptionDoc("aggregates", "Aggregate name(s) for new FlexVols (list or comma " + "separated); with several, the one with most free space is used. " + "Default: ONTAP chooses."), + OptionDoc("volume_per_lun", "Create one FlexVol per LUN/namespace.", default=True), + OptionDoc("flexvol", "Existing FlexVol for all LUNs/namespaces; required when " + "volume_per_lun is false."), + OptionDoc("space_guarantee", "FlexVol space guarantee: 'none' (thin) or " + "'volume'.", default="none"), + OptionDoc("snapshot_reserve_percent", "FlexVol snapshot reserve (0-90).", default=0), + OptionDoc("snapshot_policy", "FlexVol snapshot policy.", default="none"), + OptionDoc("flexvol_headroom_percent", "FlexVol size above the LUN/namespace size, " + "for metadata.", default=5), + OptionDoc("os_type", "os_type of LUNs, namespaces, igroups and subsystems.", + default="linux"), + OptionDoc("namespace_block_size", "NVMe namespace block size (512 or 4096).", + default=4096), + OptionDoc("job_timeout_s", "How long to wait for an asynchronous ONTAP job.", + default=300), + OptionDoc("skip_recovery_queue", "Delete FlexVols with force=true so they do not " + "linger (and use space) in the volume recovery queue.", default=False), + ) + + def __init__(self, ctx): + super().__init__(ctx) + self._sleep: Callable[[float], None] = self.session.sleep + self.svm = str(self.option("svm")).strip() + if not self.svm: + raise ArrayError("NetApp ONTAP option svm must not be empty") + self.family = str(self.option("protocol_family")).strip().lower() + if self.family not in _FAMILIES: + raise ArrayError(f"NetApp ONTAP protocol_family must be one of {_FAMILIES}, " + f"got {self.family!r}") + self.volume_per_lun = _as_bool(self.option("volume_per_lun")) + self.shared_flexvol = self.option("flexvol") + if not self.volume_per_lun and not self.shared_flexvol: + raise ArrayError("NetApp ONTAP option flexvol is required when volume_per_lun " + "is false") + aggr = self.option("aggregates") or [] + if isinstance(aggr, str): + aggr = [a for a in (x.strip() for x in aggr.split(",")) if a] + self.aggregates: list[str] = [str(a) for a in aggr] + self.guarantee = str(self.option("space_guarantee")) + if self.guarantee not in _GUARANTEES: + raise ArrayError(f"NetApp ONTAP space_guarantee must be one of {_GUARANTEES}") + self.snapshot_reserve = self._int_option("snapshot_reserve_percent", 0, 90) + self.headroom = self._int_option("flexvol_headroom_percent", 0, 100) + self.block_size = self._int_option("namespace_block_size", 512, 4096) + if self.block_size not in (512, 4096): + raise ArrayError("NetApp ONTAP namespace_block_size must be 512 or 4096") + self.job_timeout_s = float(self._int_option("job_timeout_s", 1, 86400)) + self.os_type = str(self.option("os_type")) + self.skip_recovery_queue = _as_bool(self.option("skip_recovery_queue")) + + def _int_option(self, name: str, lo: int, hi: int) -> int: + raw = self.option(name) + try: + value = int(raw) + except (TypeError, ValueError): + raise ArrayError(f"NetApp ONTAP option {name} must be an integer") from None + if not lo <= value <= hi: + raise ArrayError(f"NetApp ONTAP option {name} must be in {lo}..{hi}") + return value + + # -- session ------------------------------------------------------------ + + def login(self) -> None: + self.session.set_basic_auth(self.secret("username"), self.secret("password")) + if not self._records("/api/svm/svms", {"name": self.svm, "fields": "uuid,name"}): + raise ArrayNotFoundError(f"SVM {self.svm!r} not found on array {self.config.id}") + if self._is_disaggregated(): + raise ArrayUnsupportedError( + f"Array {self.config.id} is an ONTAP ASA r2 (disaggregated) system, which has " + "no FlexVols; the NetApp ONTAP adapter supports unified ONTAP and classic ASA") + + def _is_disaggregated(self) -> bool: + """ASA r2 check as in Trident: ``disaggregated``, asked for on 9.16.1+.""" + cluster = json_of(self._call("GET", "/api/cluster", params={"fields": "version"})) or {} + ver = cluster.get("version") or {} + try: + parsed = (int(ver["generation"]), int(ver["major"]), int(ver["minor"])) + except (KeyError, TypeError, ValueError): + return False + if parsed < _ASA_R2_MIN_VERSION: + return False + info = json_of(self._call("GET", "/api/cluster", + params={"fields": "disaggregated"})) or {} + return _as_bool(info.get("disaggregated", False)) + + def close(self) -> None: + self.session.clear_auth() + + # -- request helpers ----------------------------------------------------- + + def _call(self, method: str, path: str, *, params: dict[str, Any] | None = None, + json: Any = None) -> requests.Response: + """One request; ONTAP 400 errors are classified by message.""" + resp = self.session.request(method, path, params=params, json=json, ok=(400,)) + if resp.status_code == 400: + raise self._error(method, path, json_of(resp) or {}) + return resp + + @staticmethod + def _error(method: str, path: str, body: dict[str, Any]) -> ArrayError: + err = body.get("error") or {} + message = str(err.get("message", "")) or "request failed" + code = str(err.get("code", "")) + text = f"{method} {path}: {message} (ONTAP error {code})" + lowered = message.lower() + if code in _NOT_FOUND_CODES or "doesn't exist" in lowered \ + or "does not exist" in lowered or "not found" in lowered: + return ArrayNotFoundError(text) + if code in _CONFLICT_CODES or any( + w in lowered for w in ("already", "duplicate", "in use", "is mapped")): + return ArrayConflictError(text) + return ArrayRequestError(text, status=400) + + def _records(self, path: str, params: dict[str, Any]) -> list[dict[str, Any]]: + """GET a collection, following ``_links.next``.""" + out: list[dict[str, Any]] = [] + next_path: str | None = path + query: dict[str, Any] | None = params + while next_path: + body = json_of(self._call("GET", next_path, params=query)) or {} + out.extend(body.get("records", [])) + next_path = ((body.get("_links") or {}).get("next") or {}).get("href") + query = None + return out + + def _one(self, path: str, params: dict[str, Any]) -> dict[str, Any] | None: + records = self._records(path, params) + return records[0] if records else None + + def _async(self, method: str, path: str, *, params: dict[str, Any] | None = None, + json: Any = None) -> dict[str, Any]: + """Issue a request that may answer 202 + job, and wait for the job.""" + resp = self._call(method, path, params=params, json=json) + body = json_of(resp) or {} + job = body.get("job") + if job: + self._wait_job(job, f"{method} {path}") + return body + + def _wait_job(self, job: dict[str, Any], what: str) -> None: + href = ((job.get("_links") or {}).get("self") or {}).get("href") \ + or f"/api/cluster/jobs/{job['uuid']}" + waited = 0.0 + interval = 0.5 + while True: + state = json_of(self._call("GET", href, + params={"fields": "uuid,state,message,code"})) or {} + status = state.get("state") + if status == "success": + return + if status == "failure": + raise self._error_from_job(what, state) + if waited >= self.job_timeout_s: + raise ArrayRequestError( + f"{what}: ONTAP job {job.get('uuid')} still {status} after " + f"{self.job_timeout_s:.0f}s") + self._sleep(interval) + waited += interval + interval = min(interval * 2, 5.0) + + def _error_from_job(self, what: str, state: dict[str, Any]) -> ArrayError: + method, _, path = what.partition(" ") + message = f"{state.get('message') or 'job failed'} [job {state.get('uuid', '')}]" + return self._error(method, path, {"error": {"message": message, + "code": state.get("code", "")}}) + + @staticmethod + def _created(resp_body: dict[str, Any]) -> dict[str, Any] | None: + records = resp_body.get("records") or [] + return records[0] if records else None + + def _svm_ref(self) -> dict[str, str]: + return {"name": self.svm} + + # -- volumes: lookup ----------------------------------------------------- + + def _volume_from_lun(self, rec: dict[str, Any]) -> Volume: + serial = rec.get("serial_number") or "" + loc = rec.get("location") or {} + name = loc.get("logical_unit") or str(rec.get("name", "")).rsplit("/", 1)[-1] + return Volume(id=rec["uuid"], name=name, size_bytes=int(rec["space"]["size"]), + wwn=ontap_wwn(serial) if serial else None) + + @staticmethod + def _volume_from_ns(rec: dict[str, Any]) -> Volume: + loc = rec.get("location") or {} + name = loc.get("namespace") or str(rec.get("name", "")).rsplit("/", 1)[-1] + return Volume(id=rec["uuid"], name=name, size_bytes=int(rec["space"]["size"])) + + def _kinds(self) -> tuple[str, str]: + return (_LUN, _NS) if self.family == "scsi" else (_NS, _LUN) + + def _lookup(self, kind: str, params: dict[str, Any]) -> dict[str, Any] | None: + if kind == _LUN: + return self._one("/api/storage/luns", {**params, "svm.name": self.svm, + "fields": _LUN_FIELDS}) + return self._one("/api/storage/namespaces", {**params, "svm.name": self.svm, + "fields": _NS_FIELDS}) + + def _locate(self, volume_id: str) -> tuple[str, dict[str, Any]] | None: + for kind in self._kinds(): + rec = self._lookup(kind, {"uuid": volume_id}) + if rec is not None: + return kind, rec + return None + + def _require(self, volume_id: str) -> tuple[str, dict[str, Any]]: + found = self._locate(volume_id) + if found is None: + raise ArrayNotFoundError( + f"Volume {volume_id} not found in SVM {self.svm} on array {self.config.id}") + return found + + def _to_volume(self, kind: str, rec: dict[str, Any]) -> Volume: + return self._volume_from_lun(rec) if kind == _LUN else self._volume_from_ns(rec) + + def _flexvol_for(self, name: str) -> str: + return flexvol_name(name) if self.volume_per_lun else str(self.shared_flexvol) + + # -- contract ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + cluster = json_of(self._call("GET", "/api/cluster", + params={"fields": "name,uuid,version"})) or {} + model = "" + try: + nodes = self._records("/api/cluster/nodes", {"fields": "name,model,serial_number"}) + model = next((n["model"] for n in nodes if n.get("model")), "") + except (ArrayRequestError, ArrayNotFoundError): + pass + version = (cluster.get("version") or {}).get("full", "") + return ArrayInfo(vendor=self.vendor, model=model, version=version, + serial=cluster.get("uuid", "")) + + def get_volume(self, volume_id: str) -> Volume | None: + found = self._locate(volume_id) + return self._to_volume(*found) if found else None + + def find_volume(self, name: str) -> Volume | None: + path = f"/vol/{self._flexvol_for(name)}/{name}" + for kind in self._kinds(): + rec = self._lookup(kind, {"name": path}) + if rec is not None: + return self._to_volume(kind, rec) + return None + + def _flexvol_bytes(self, size_bytes: int) -> int: + usable = -(-size_bytes * (100 + self.headroom) // 100) + total = -(-usable * 100 // (100 - self.snapshot_reserve)) + total = -(-total // 4096) * 4096 + return max(total, MIN_FLEXVOL_BYTES) + + def _get_flexvol(self, name: str) -> dict[str, Any] | None: + return self._one("/api/storage/volumes", + {"name": name, "svm.name": self.svm, "fields": "uuid,name,size"}) + + def _pick_aggregate(self) -> list[dict[str, str]]: + if not self.aggregates: + return [] + if len(self.aggregates) == 1: + return [{"name": self.aggregates[0]}] + # More than one aggregate in the POST would make a FlexGroup: choose one. + recs = self._records("/api/storage/aggregates", + {"name": "|".join(self.aggregates), + "fields": "name,space.block_storage.available"}) + if not recs: + raise ArrayNotFoundError(f"None of the aggregates {self.aggregates} exist") + best = max(recs, key=lambda r: int( + ((r.get("space") or {}).get("block_storage") or {}).get("available", 0))) + return [{"name": best["name"]}] + + def _create_flexvol(self, name: str, size_bytes: int) -> None: + body: dict[str, Any] = { + "name": name, + "svm": self._svm_ref(), + "size": size_bytes, + "type": "rw", + "guarantee": {"type": self.guarantee}, + "space": {"snapshot": {"reserve_percent": self.snapshot_reserve}}, + "snapshot_policy": {"name": str(self.option("snapshot_policy"))}, + "comment": FLEXVOL_COMMENT, + } + aggrs = self._pick_aggregate() + if aggrs: + body["aggregates"] = aggrs + self._async("POST", "/api/storage/volumes", json=body) + + def _grow_flexvol(self, vol: dict[str, Any], size_bytes: int) -> None: + if int(vol.get("size", 0)) < size_bytes: + self._async("PATCH", f"/api/storage/volumes/{vol['uuid']}", + json={"size": size_bytes}) + + def _delete_flexvol(self, uuid: str) -> None: + try: + self._async("DELETE", f"/api/storage/volumes/{uuid}", + params={"force": "true"} if self.skip_recovery_queue else None) + except ArrayNotFoundError: + pass + + def create_volume(self, name: str, size_bytes: int) -> Volume: + flex = self._flexvol_for(name) + created_flexvol: str | None = None + if self.volume_per_lun: + want = self._flexvol_bytes(size_bytes) + vol = self._get_flexvol(flex) + if vol is None: # a re-run may find the FlexVol of an interrupted create + self._create_flexvol(flex, want) + vol = self._get_flexvol(flex) + if vol is None: + raise ArrayRequestError(f"FlexVol {flex} missing after create") + created_flexvol = vol["uuid"] + else: + self._grow_flexvol(vol, want) + path = f"/vol/{flex}/{name}" + kind = _LUN if self.family == "scsi" else _NS + try: + if kind == _LUN: + self._async("POST", "/api/storage/luns", params={"return_records": "true"}, json={ + "svm": self._svm_ref(), "name": path, "os_type": self.os_type, + "space": {"size": size_bytes, "guarantee": {"requested": False}}, + }) + else: + self._async("POST", "/api/storage/namespaces", + params={"return_records": "true"}, + json={"svm": self._svm_ref(), "name": path, "os_type": self.os_type, + "space": {"size": size_bytes, + "block_size": self.block_size}}) + except ArrayError: + if created_flexvol: + try: + self._delete_flexvol(created_flexvol) + except ArrayError: + pass + raise + rec = self._lookup(kind, {"name": path}) + if rec is None: + raise ArrayRequestError(f"{kind} {path} missing after create") + return self._to_volume(kind, rec) + + def _owns_flexvol(self, kind: str, rec: dict[str, Any]) -> bool: + """True when the LUN/namespace sits in its own per-volume FlexVol.""" + flex = ((rec.get("location") or {}).get("volume") or {}).get("name", "") + name = self._to_volume(kind, rec).name + return bool(flex) and flex != self.shared_flexvol and flex == flexvol_name(name) + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + kind, rec = self._require(volume_id) + loc_vol = (rec.get("location") or {}).get("volume") or {} + if self._owns_flexvol(kind, rec): + vol = self._get_flexvol(loc_vol.get("name", "")) + if vol is None: + raise ArrayNotFoundError(f"FlexVol of volume {volume_id} not found") + self._grow_flexvol(vol, self._flexvol_bytes(size_bytes)) + base = "/api/storage/luns" if kind == _LUN else "/api/storage/namespaces" + self._async("PATCH", f"{base}/{volume_id}", json={"space": {"size": size_bytes}}) + kind, rec = self._require(volume_id) + return self._to_volume(kind, rec) + + def delete_volume(self, volume_id: str) -> None: + kind, rec = self._require(volume_id) + loc_vol = (rec.get("location") or {}).get("volume") or {} + if kind == _LUN: + self._async("DELETE", f"/api/storage/luns/{volume_id}") + else: + smap = self._subsystem_map(volume_id) + if smap is not None: + sub = smap["subsystem"] + self._call("DELETE", + f"/api/protocols/nvme/subsystem-maps/{sub['uuid']}/{volume_id}") + self._delete_subsystem(sub["uuid"]) + self._async("DELETE", f"/api/storage/namespaces/{volume_id}") + # Only a FlexVol of our own (one per LUN) is removed; never the shared one. + flex = loc_vol.get("name", "") + if self._owns_flexvol(kind, rec): + uuid = loc_vol.get("uuid") or (self._get_flexvol(flex) or {}).get("uuid") + if uuid: + self._delete_flexvol(uuid) + + # -- hosts ---------------------------------------------------------------- + + @staticmethod + def _nqn(i: Initiator) -> str: + if i.protocol == Protocol.NVME_FC and ":" in i.address and not i.address.startswith( + "nqn."): + raise ArrayUnsupportedError( + "ONTAP identifies NVMe/FC hosts by host NQN, not by WWPN: " + f"{i.address!r}") + return i.address + + def ensure_host(self, spec: HostSpec) -> str: + if all(i.protocol.is_nvme for i in spec.initiators): + return self._ensure_nvme_host(spec) + if all(i.protocol.is_scsi for i in spec.initiators): + return self._ensure_igroup(spec) + raise ArrayUnsupportedError( + f"NetApp ONTAP adapter does not support {sorted(p.value for p in spec.protocols)}") + + # SCSI: igroups + + @staticmethod + def _igroup_initiator(i: Initiator) -> str: + return colon_wwpn(i.address) if i.protocol == Protocol.FC else i.normalized() + + @staticmethod + def _norm_initiator(value: str) -> str: + v = value.strip().lower() + if v.startswith(("iqn.", "eui.", "naa.")): + return v + return v.replace(":", "").replace("-", "").removeprefix("0x") + + def _igroup(self, name: str) -> dict[str, Any] | None: + return self._one("/api/protocols/san/igroups", + {"name": name, "svm.name": self.svm, + "fields": "uuid,name,protocol,os_type,initiators.name"}) + + def _ensure_igroup(self, spec: HostSpec) -> str: + wanted = [self._igroup_initiator(i) for i in spec.initiators] + owners = self._records("/api/protocols/san/igroups", + {"svm.name": self.svm, "initiators.name": "|".join(wanted), + "fields": "uuid,name,initiators.name"}) + want_norm = {self._norm_initiator(w) for w in wanted} + for g in owners: + if g["name"] == spec.name: + continue + taken = want_norm & {self._norm_initiator(i["name"]) + for i in g.get("initiators") or []} + if taken: + raise ArrayConflictError( + f"Initiator(s) {', '.join(sorted(taken))} already belong to igroup " + f"{g['name']!r} in SVM {self.svm}") + group = self._igroup(spec.name) + if group is None: + self._call("POST", "/api/protocols/san/igroups", + params={"return_records": "true"}, + json={"svm": self._svm_ref(), "name": spec.name, "os_type": self.os_type, + "protocol": "mixed", + "initiators": [{"name": w} for w in wanted]}) + return spec.name + have = {self._norm_initiator(i["name"]) for i in group.get("initiators") or []} + missing = [w for w in wanted if self._norm_initiator(w) not in have] + if missing: + needs = {"iscsi" if i.protocol == Protocol.ISCSI else "fcp" + for i in spec.initiators + if self._norm_initiator(self._igroup_initiator(i)) not in have} + proto = group.get("protocol") or "mixed" + if proto != "mixed" and needs - {proto}: + # "The protocol of an initiator group cannot be changed after + # creation of the group." (ONTAP REST spec, igroup.protocol) + raise ArrayConflictError( + f"igroup {spec.name!r} in SVM {self.svm} has protocol {proto!r} and " + f"cannot take {', '.join(sorted(needs - {proto}))} initiators; ONTAP " + "does not allow changing an igroup's protocol") + self._call("POST", f"/api/protocols/san/igroups/{group['uuid']}/initiators", + json={"records": [{"name": m} for m in missing]}) + return spec.name + + # NVMe: host subsystems + + def _subsystems(self, params: dict[str, Any]) -> list[dict[str, Any]]: + return self._records("/api/protocols/nvme/subsystems", + {**params, "svm.name": self.svm, + "fields": "uuid,name,comment,os_type,hosts.nqn"}) + + @staticmethod + def _is_volume_subsystem(sub: dict[str, Any]) -> bool: + return sub.get("comment") == VOLUME_SUBSYSTEM_COMMENT + + def _host_subsystem(self, name: str) -> dict[str, Any] | None: + subs = [s for s in self._subsystems({"name": name}) if not self._is_volume_subsystem(s)] + return subs[0] if subs else None + + def _ensure_nvme_host(self, spec: HostSpec) -> str: + wanted = [self._nqn(i) for i in spec.initiators] + for other in self._subsystems({"hosts.nqn": "|".join(wanted)}): + if other["name"] == spec.name or self._is_volume_subsystem(other): + continue + taken = set(wanted) & {h["nqn"] for h in other.get("hosts") or []} + if taken: + raise ArrayConflictError( + f"Host NQN(s) {', '.join(sorted(taken))} already belong to subsystem " + f"{other['name']!r} in SVM {self.svm}") + sub = self._host_subsystem(spec.name) + if sub is None: + self._call("POST", "/api/protocols/nvme/subsystems", + params={"return_records": "true"}, + json={"svm": self._svm_ref(), "name": spec.name, "os_type": self.os_type, + "hosts": [{"nqn": n} for n in wanted]}) + return spec.name + have = {h["nqn"] for h in sub.get("hosts") or []} + missing = [n for n in wanted if n not in have] + if missing: + self._add_subsystem_hosts(sub["uuid"], missing) + return spec.name + + def _add_subsystem_hosts(self, sub_uuid: str, nqns: list[str]) -> None: + self._call("POST", f"/api/protocols/nvme/subsystems/{sub_uuid}/hosts", + json={"records": [{"nqn": n} for n in nqns]}) + + def _delete_subsystem(self, sub_uuid: str) -> None: + try: + self._call("DELETE", f"/api/protocols/nvme/subsystems/{sub_uuid}", + params={"allow_delete_with_hosts": "true"}) + except ArrayNotFoundError: + pass + + def delete_host(self, host_id: str) -> None: + group = self._igroup(host_id) + if group is not None: + try: + self._call("DELETE", f"/api/protocols/san/igroups/{group['uuid']}") + except ArrayNotFoundError: + pass + sub = self._host_subsystem(host_id) + if sub is not None: + self._delete_subsystem(sub["uuid"]) + + # -- mappings ------------------------------------------------------------- + + def _lun_maps(self, lun_uuid: str, igroup: str | None = None) -> list[dict[str, Any]]: + params: dict[str, Any] = {"lun.uuid": lun_uuid, "svm.name": self.svm, + "fields": "lun.uuid,igroup.uuid,igroup.name," + "logical_unit_number"} + if igroup is not None: + params["igroup.name"] = igroup + return self._records("/api/protocols/san/lun-maps", params) + + @staticmethod + def _lun_mapping(rec: dict[str, Any]) -> Mapping: + return Mapping(volume_id=rec["lun"]["uuid"], host_id=rec["igroup"]["name"], + lun=rec.get("logical_unit_number")) + + def _subsystem_map(self, ns_uuid: str) -> dict[str, Any] | None: + return self._one("/api/protocols/nvme/subsystem-maps", + {"namespace.uuid": ns_uuid, "svm.name": self.svm, + "fields": "namespace.uuid,subsystem.uuid,subsystem.name,nsid"}) + + def _volume_subsystem_hosts(self, smap: dict[str, Any]) -> dict[str, Any]: + sub = smap["subsystem"] + recs = self._subsystems({"uuid": sub["uuid"]}) + if not recs: + raise ArrayNotFoundError(f"Subsystem {sub.get('name')} not found") + return recs[0] + + def _nvme_mappings(self, ns_uuid: str) -> list[Mapping]: + smap = self._subsystem_map(ns_uuid) + if smap is None: + return [] + vsub = self._volume_subsystem_hosts(smap) + nqns = {h["nqn"] for h in vsub.get("hosts") or []} + if not nqns: + return [] + nsid = parse_nsid(smap.get("nsid")) + out: list[Mapping] = [] + for sub in self._subsystems({"hosts.nqn": "|".join(sorted(nqns))}): + if self._is_volume_subsystem(sub): + continue + if nqns & {h["nqn"] for h in sub.get("hosts") or []}: + out.append(Mapping(volume_id=ns_uuid, host_id=sub["name"], lun=nsid)) + return out + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + kind, rec = self._require(volume_id) + if kind == _LUN: + return self._map_lun(volume_id, host_id) + return self._map_namespace(volume_id, rec, host_id) + + def _map_lun(self, lun_uuid: str, host_id: str) -> Mapping: + if self._igroup(host_id) is None: + if self._host_subsystem(host_id) is not None: + raise ArrayUnsupportedError( + f"Host {host_id} is an NVMe host; volume {lun_uuid} is a SCSI LUN " + "(protocol_family scsi)") + raise ArrayNotFoundError(f"igroup {host_id} not found in SVM {self.svm}") + existing = self._lun_maps(lun_uuid, host_id) + if existing: + return self._lun_mapping(existing[0]) + self._call("POST", "/api/protocols/san/lun-maps", params={"return_records": "true"}, + json={"svm": self._svm_ref(), "lun": {"uuid": lun_uuid}, + "igroup": {"name": host_id}}) + created = self._lun_maps(lun_uuid, host_id) + if not created: + raise ArrayRequestError(f"lun-map {lun_uuid} -> {host_id} missing after create") + return self._lun_mapping(created[0]) + + def _map_namespace(self, ns_uuid: str, rec: dict[str, Any], host_id: str) -> Mapping: + host = self._host_subsystem(host_id) + if host is None: + if self._igroup(host_id) is not None: + raise ArrayUnsupportedError( + f"Host {host_id} is a SCSI host; volume {ns_uuid} is an NVMe namespace " + "(protocol_family nvme)") + raise ArrayNotFoundError(f"NVMe host subsystem {host_id} not found in SVM {self.svm}") + host_nqns = [h["nqn"] for h in host.get("hosts") or []] + smap = self._subsystem_map(ns_uuid) + if smap is None: + sub_name = flexvol_name(self._volume_from_ns(rec).name) + vsubs = [s for s in self._subsystems({"name": sub_name}) + if self._is_volume_subsystem(s)] + if vsubs: + sub_uuid = vsubs[0]["uuid"] + else: + body = json_of(self._call( + "POST", "/api/protocols/nvme/subsystems", params={"return_records": "true"}, + json={"svm": self._svm_ref(), "name": sub_name, "os_type": self.os_type, + "comment": VOLUME_SUBSYSTEM_COMMENT})) or {} + created = self._created(body) + if created is None: + created = next(iter(s for s in self._subsystems({"name": sub_name}) + if self._is_volume_subsystem(s)), None) + if created is None: + raise ArrayRequestError(f"Subsystem {sub_name} missing after create") + sub_uuid = created["uuid"] + self._call("POST", "/api/protocols/nvme/subsystem-maps", + params={"return_records": "true"}, + json={"svm": self._svm_ref(), "namespace": {"uuid": ns_uuid}, + "subsystem": {"uuid": sub_uuid}}) + smap = self._subsystem_map(ns_uuid) + if smap is None: + raise ArrayRequestError(f"subsystem-map of {ns_uuid} missing after create") + vsub = self._volume_subsystem_hosts(smap) + if not self._is_volume_subsystem(vsub): + raise ArrayConflictError( + f"Namespace {ns_uuid} is mapped to subsystem {vsub['name']!r}, which " + "simplyblock does not manage") + have = {h["nqn"] for h in vsub.get("hosts") or []} + missing = [n for n in host_nqns if n not in have] + if missing: + self._add_subsystem_hosts(vsub["uuid"], missing) + return Mapping(volume_id=ns_uuid, host_id=host_id, lun=parse_nsid(smap.get("nsid"))) + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + found = self._locate(volume_id) + if found is None: + return + kind, _ = found + if kind == _LUN: + for m in self._lun_maps(volume_id, host_id): + try: + self._call("DELETE", f"/api/protocols/san/lun-maps/{volume_id}/" + f"{m['igroup']['uuid']}") + except ArrayNotFoundError: + pass + return + smap = self._subsystem_map(volume_id) + host = self._host_subsystem(host_id) + if smap is None or host is None: + return + vsub = self._volume_subsystem_hosts(smap) + if not self._is_volume_subsystem(vsub): + return + have = {h["nqn"] for h in vsub.get("hosts") or []} + for h in host.get("hosts") or []: + if h["nqn"] in have: + try: + self._call("DELETE", f"/api/protocols/nvme/subsystems/{vsub['uuid']}/hosts/" + f"{quote(h['nqn'], safe='')}") + except ArrayNotFoundError: + pass + + def list_mappings(self, volume_id: str) -> list[Mapping]: + kind, _ = self._require(volume_id) + if kind == _LUN: + return [self._lun_mapping(r) for r in self._lun_maps(volume_id)] + return self._nvme_mappings(volume_id) diff --git a/simplyblock_core/storage_arrays/adapters/pure.py b/simplyblock_core/storage_arrays/adapters/pure.py new file mode 100644 index 0000000000..676f9c17d6 --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/pure.py @@ -0,0 +1,273 @@ +"""Pure Storage FlashArray, REST API 2.x (Purity//FA 6.x and later). + +Auth: an API token (``api_token`` credential field) exchanged for a session +token via ``POST /api/2.x/login`` (``api-token`` header -> ``x-auth-token``). + +Objects are addressed by name (hosts) and id (volumes). Pure "destroys" +volumes into a recycle bin (recoverable for the eradication delay, 24 h by +default) before eradication; ``eradicate=true`` removes them at once. + +Volume identity: SCSI ``naa.624a9370``; NVMe NGUID +``eui.0024a937``. A connection's ``lun`` is set +for SCSI hosts only; NVMe hosts get ``nsid`` instead (REST 2.27+), which the +adapter reports as :attr:`Mapping.lun`. + +Checked against vendor source +----------------------------- +* Login: ``POST /api//login`` with header ``api-token``, session token + in the ``x-auth-token`` response header; ``POST .../logout`` ends it + (py-pure-client ``pypureclient/api_token_manager.py``, ``keywords.py`` + ``Headers``). +* Version discovery: ``GET /api/api_version`` -> ``{"version": [...]}`` + (py-pure-client ``pypureclient/_helpers.py`` ``get_target_versions``). +* ``POST /volumes?names=`` with body ``provisioned``; ``PATCH /volumes`` + (``ids``/``names``) with ``provisioned`` / ``destroyed``; ``DELETE + /volumes`` eradicates only destroyed volumes; ``GET /volumes`` without + ``destroyed=`` lists destroyed volumes too (py-pure-client + ``_common/apis/volumes_api_v_2_36.py``, ``models/volume_post_v_2_33.py``, + ``volume_patch_v_2_33.py``, ``volume_v_2_34.py``). +* ``GET/POST/DELETE /connections`` take ``host_names`` and ``volume_ids``; + items carry ``host.name``, ``volume.id``, ``lun`` and ``nsid`` + (``connections_api_v_2_36.py``, ``models/connection_v_2_27.py``). +* Hosts: ``iqns``/``wwns``/``nqns`` on POST, ``add_iqns``/``add_wwns``/ + ``add_nqns`` on PATCH, ``names`` on every call (``hosts_api_v_2_36.py``, + ``models/host_post_v_2_16.py``, ``host_patch_v_2_16.py``); WWNs are sent + as 16 hex digits without colons, upper case (Ansible + ``purestorage.flasharray`` ``plugins/modules/purefa_host.py``). +* ``naa.624a9370`` and ``eui.0024a937`` + (Ansible ``purefa_volume.py`` ``PURE_OUI`` / ``_create_nguid``; Cinder + ``volume/drivers/pure.py`` ``_get_wwn`` / ``_get_nguid``). +* Errors are HTTP 400 with ``{"errors": [{"message": ...}]}``; missing + objects say "does not exist", duplicates "already exists" (Cinder + ``pure.py`` ``ERR_MSG_*``; ``models/error_context_response_v_2_38.py``). + +Verify against a real system +---------------------------- +* Whether ``PATCH destroyed=true`` is refused for a volume that still has + host connections (the fake refuses; the contract unmaps first anyway). +* Whether REST 2.4..2.26 report NVMe connections in ``lun`` (``nsid`` was + added in 2.27). +""" +from __future__ import annotations + +from typing import Any + +from ..base import ( + ArrayAdapter, + ArrayConflictError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, +) +from ..http import json_of + +_SUPPORTED_MAX = (2, 36) # newest 2.x minor this adapter was written against +_MIN_VERSION = (2, 4) + + +def _ver(v: str) -> tuple[int, int]: + major, _, minor = v.partition(".") + return int(major), int(minor or 0) + + +def pure_wwn(serial: str) -> str: + return "naa.624a9370" + serial.lower() + + +def pure_nguid(serial: str) -> str: + s = serial.lower() + return "eui.00" + s[:14] + "24a937" + s[14:] + + +class PureFlashArrayAdapter(ArrayAdapter): + vendor = "pure-flasharray" + display_name = "Pure Storage FlashArray" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC, Protocol.NVME_TCP, + Protocol.NVME_RDMA, Protocol.NVME_FC}), + size_granularity_bytes=512, + min_size_bytes=1 << 20, + max_name_len=63, + ) + credential_fields = (frozenset({"api_token"}),) + option_docs = ( + OptionDoc("eradicate", "Eradicate deleted volumes immediately instead of " + "leaving them recoverable for the eradication delay.", default=False), + OptionDoc("api_version", "Pin the REST 2.x version (e.g. '2.21'); " + "default: newest supported by both sides."), + ) + + def __init__(self, ctx): + super().__init__(ctx) + self._api = "" + + # -- session ------------------------------------------------------------ + + def login(self) -> None: + version = self.option("api_version") + if not version: + offered = json_of(self.session.get("/api/api_version")).get("version", []) + usable = [v for v in offered + if v.startswith("2.") and _MIN_VERSION <= _ver(v) <= _SUPPORTED_MAX] + if not usable: + raise ArrayRequestError( + f"FlashArray offers no REST 2.x version in " + f"{'.'.join(map(str, _MIN_VERSION))}..{'.'.join(map(str, _SUPPORTED_MAX))}: " + f"{offered}") + version = max(usable, key=_ver) + self._api = f"/api/{version}" + self.session.headers.pop("x-auth-token", None) + resp = self.session.post(f"{self._api}/login", + headers={"api-token": self.secret("api_token")}) + token = resp.headers.get("x-auth-token") + if not token: + raise ArrayRequestError("FlashArray login returned no x-auth-token") + self.session.headers["x-auth-token"] = token + + def close(self) -> None: + if self._api and "x-auth-token" in self.session.headers: + try: + self.session.post(f"{self._api}/logout") + except Exception: + pass + + # -- request helpers ----------------------------------------------------- + + def _items(self, method: str, path: str, *, params=None, json=None, + missing_ok: bool = False) -> list[dict[str, Any]]: + resp = self.session.request(method, f"{self._api}{path}", params=params, json=json, + ok=(400,)) + body = json_of(resp) or {} + if resp.status_code == 400: + messages = " ".join( + str(e.get("message", "")) for e in body.get("errors", [])) + lowered = messages.lower() + if "does not exist" in lowered or "not found" in lowered: + if missing_ok: + return [] + raise ArrayNotFoundError(f"{method} {path}: {messages}") + if "already" in lowered or "in use" in lowered or "exists" in lowered: + raise ArrayConflictError(f"{method} {path}: {messages}") + raise ArrayRequestError(f"{method} {path}: {messages}", status=400) + return list(body.get("items", [])) + + @staticmethod + def _volume(item: dict[str, Any]) -> Volume: + serial = item.get("serial") or "" + return Volume( + id=item["id"], name=item["name"], size_bytes=int(item["provisioned"]), + wwn=pure_wwn(serial) if serial else None, + nguid=pure_nguid(serial) if serial else None, + ) + + # -- contract ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + arr = self._items("GET", "/arrays")[0] + model = "" + try: + controllers = self._items("GET", "/controllers") + model = next((c.get("model", "") for c in controllers if c.get("model")), "") + except ArrayRequestError: + pass + return ArrayInfo(vendor=self.vendor, model=model, + version=f"{arr.get('os', 'Purity//FA')} {arr.get('version', '')}".strip(), + serial=arr.get("id", "")) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + items = self._items("POST", "/volumes", params={"names": name}, + json={"provisioned": size_bytes}) + return self._volume(items[0]) + + def _live(self, items: list[dict[str, Any]]) -> Volume | None: + live = [i for i in items if not i.get("destroyed")] + return self._volume(live[0]) if live else None + + def get_volume(self, volume_id: str) -> Volume | None: + return self._live(self._items("GET", "/volumes", params={"ids": volume_id}, + missing_ok=True)) + + def find_volume(self, name: str) -> Volume | None: + return self._live(self._items("GET", "/volumes", params={"names": name}, + missing_ok=True)) + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + items = self._items("PATCH", "/volumes", params={"ids": volume_id}, + json={"provisioned": size_bytes}) + return self._volume(items[0]) + + def delete_volume(self, volume_id: str) -> None: + self._items("PATCH", "/volumes", params={"ids": volume_id}, json={"destroyed": True}) + if self.option("eradicate"): + self._items("DELETE", "/volumes", params={"ids": volume_id}) + + def _host(self, name: str) -> dict[str, Any] | None: + items = self._items("GET", "/hosts", params={"names": name}, missing_ok=True) + return items[0] if items else None + + def ensure_host(self, spec: HostSpec) -> str: + wanted = { + "iqns": spec.addresses(Protocol.ISCSI), + "wwns": [a.upper() for a in spec.addresses(Protocol.FC)], + "nqns": spec.addresses(Protocol.NVME_TCP, Protocol.NVME_RDMA, Protocol.NVME_FC), + } + wanted = {k: v for k, v in wanted.items() if v} + host = self._host(spec.name) + if host is None: + self._items("POST", "/hosts", params={"names": spec.name}, json=wanted) + return spec.name + add: dict[str, list[str]] = {} + for key, values in wanted.items(): + have = {str(v).lower() for v in host.get(key, [])} + missing = [v for v in values if v.lower() not in have] + if missing: + add[f"add_{key}"] = missing + if add: + self._items("PATCH", "/hosts", params={"names": spec.name}, json=add) + return spec.name + + def delete_host(self, host_id: str) -> None: + if self._host(host_id) is None: + return + if self._items("GET", "/connections", params={"host_names": host_id}, missing_ok=True): + raise ArrayConflictError(f"Host {host_id} still has volume connections") + self._items("DELETE", "/hosts", params={"names": host_id}, missing_ok=True) + + def _connections(self, volume_id: str, host_id: str | None = None) -> list[Mapping]: + params: dict[str, str] = {"volume_ids": volume_id} + if host_id is not None: + params["host_names"] = host_id + items = self._items("GET", "/connections", params=params, missing_ok=True) + return [self._mapping(i) for i in items if i.get("host")] + + @staticmethod + def _mapping(item: dict[str, Any]) -> Mapping: + # ``lun`` is set for SCSI connections only, ``nsid`` for NVMe ones. + number = item.get("lun") + if number is None: + number = item.get("nsid") + return Mapping(volume_id=item["volume"]["id"], host_id=item["host"]["name"], + lun=number) + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + existing = self._connections(volume_id, host_id) + if existing: + return existing[0] + items = self._items("POST", "/connections", + params={"host_names": host_id, "volume_ids": volume_id}) + return self._mapping(items[0]) + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + if not self._connections(volume_id, host_id): + return + self._items("DELETE", "/connections", + params={"host_names": host_id, "volume_ids": volume_id}, missing_ok=True) + + def list_mappings(self, volume_id: str) -> list[Mapping]: + return self._connections(volume_id) diff --git a/simplyblock_core/storage_arrays/base.py b/simplyblock_core/storage_arrays/base.py new file mode 100644 index 0000000000..c289df84cc --- /dev/null +++ b/simplyblock_core/storage_arrays/base.py @@ -0,0 +1,404 @@ +"""Vendor-neutral model and adapter contract for external storage arrays. + +An adapter translates five operations -- volume create/resize/delete, host +(initiator group) management and volume-to-host mapping -- into one vendor's +management API. Everything vendor-neutral (credential resolution, +re-authentication, size rounding, idempotence, delete/resize safety checks) +lives in :class:`simplyblock_core.storage_arrays.client.ArrayClient`, so +adapters stay thin and are tested against one shared contract suite +(``tests/unit/storage_arrays/test_contract.py``). +""" +from __future__ import annotations + +import json +import re +from abc import ABC, abstractmethod +from collections.abc import Mapping as MappingABC +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, ClassVar + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from simplyblock_core.credentials import Credential + +# -- Errors ----------------------------------------------------------------- + + +class ArrayError(Exception): + """Base class. Messages never contain credential values.""" + + +class ArrayAuthError(ArrayError): + """The array rejected the credential or the session expired.""" + + +class ArrayNotFoundError(ArrayError): + """The addressed object does not exist.""" + + +class ArrayConflictError(ArrayError): + """The request conflicts with array state (name taken, initiator owned + by another host, volume still mapped, ...).""" + + +class ArrayUnsupportedError(ArrayError): + """The adapter or array does not support the requested operation.""" + + +class ArrayRequestError(ArrayError): + """Any other failed request.""" + + def __init__(self, message: str, status: int | None = None): + super().__init__(message) + self.status = status + + +# -- Model ------------------------------------------------------------------ + + +class Protocol(str, Enum): + ISCSI = "iscsi" + FC = "fc" + NVME_TCP = "nvme-tcp" + NVME_RDMA = "nvme-rdma" + NVME_FC = "nvme-fc" + #: Dell PowerFlex native client (SDC); the initiator address is the SDC GUID. + SDC = "sdc" + + @property + def is_nvme(self) -> bool: + return self in (Protocol.NVME_TCP, Protocol.NVME_RDMA, Protocol.NVME_FC) + + @property + def is_scsi(self) -> bool: + return self in (Protocol.ISCSI, Protocol.FC) + + +_WWPN_RE = re.compile(r"^[0-9a-f]{16}$") + + +def normalize_wwpn(value: str) -> str: + """``21:00:00:24:FF:4C:1A:2B`` / ``0x2100...`` -> ``21000024ff4c1a2b``.""" + v = value.strip().lower().removeprefix("0x").replace(":", "").replace("-", "") + if not _WWPN_RE.match(v): + raise ValueError(f"Not a WWPN: {value!r}") + return v + + +def colon_wwpn(value: str) -> str: + """``21000024ff4c1a2b`` -> ``21:00:00:24:ff:4c:1a:2b``.""" + v = normalize_wwpn(value) + return ":".join(v[i:i + 2] for i in range(0, 16, 2)) + + +class Initiator(BaseModel): + """One host port: an iSCSI IQN, an FC WWPN, an NVMe host NQN or an SDC GUID.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + protocol: Protocol + address: str = Field(min_length=1) + + @field_validator("address") + @classmethod + def _strip(cls, v: str) -> str: + return v.strip() + + def normalized(self) -> str: + # NVMe/FC hosts are identified by their host NQN like every NVMe + # transport; only SCSI over FC uses WWPNs. + if self.protocol == Protocol.FC: + return normalize_wwpn(self.address) + if self.protocol == Protocol.ISCSI: + return self.address.lower() + return self.address + + +class HostSpec(BaseModel): + """A host as the array should know it. All initiators share one protocol + family (SCSI or NVMe); a host using both is registered twice.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str = Field(min_length=1, max_length=255) + initiators: tuple[Initiator, ...] = Field(min_length=1) + + @field_validator("initiators") + @classmethod + def _one_family(cls, v: tuple[Initiator, ...]) -> tuple[Initiator, ...]: + protocols = {i.protocol for i in v} + if len(protocols) > 1 and not ( + all(p.is_scsi for p in protocols) or all(p.is_nvme for p in protocols) + ): + raise ValueError("A host's initiators must all be SCSI or all be NVMe") + return v + + @property + def protocols(self) -> frozenset[Protocol]: + return frozenset(i.protocol for i in self.initiators) + + def addresses(self, *protocols: Protocol) -> list[str]: + return [i.normalized() for i in self.initiators if i.protocol in protocols] + + +class Volume(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + #: The array's own handle for the volume (id, uuid, device number, name). + id: str + name: str + size_bytes: int + #: SCSI identity (VPD page 0x83 designator) as the host sees it: + #: ``naa.`` for most arrays, ``eui.`` where the array presents an + #: EUI-64 designator (HPE Nimble). Lowercase. This matches the array volume + #: to ``/dev/disk/by-id`` / the multipath WWID on the host. + wwn: str | None = None + #: NVMe identity, ``eui.<32 hex>`` (NGUID) where the array exposes one. + nguid: str | None = None + + @property + def identifier(self) -> str: + """Stable identity used for safety checks: WWN, else NGUID, else id.""" + return self.wwn or self.nguid or f"id.{self.id}" + + +class Mapping(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + volume_id: str + host_id: str + #: SCSI LUN or NVMe namespace ID as presented to the host; ``None`` where + #: the protocol has no such number (PowerFlex SDC). + lun: int | None = None + + +class ArrayInfo(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + vendor: str + model: str = "" + version: str = "" + serial: str = "" + + +class ArrayConfig(BaseModel): + """One registered array. Holds no secret: ``credential_ref`` is resolved + through :class:`simplyblock_core.credentials.CredentialProvider`.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + id: str = Field(min_length=1) + vendor: str = Field(min_length=1) + #: Management endpoint, e.g. ``https://pure01.example.com`` or + #: ``https://unisphere.example.com:8443``. Path prefixes are kept. + endpoint: str = Field(min_length=1) + credential_ref: dict[str, Any] + #: PEM CA bundle (or pinned self-signed certificate) for the endpoint. + ca_bundle: str | None = None + #: Must stay True outside of labs; ``False`` disables TLS verification. + verify_tls: bool = True + timeout_s: float = Field(default=60.0, gt=0) + #: Vendor-specific settings (pool, SVM, CPG, storage group, ...); each + #: adapter documents what it reads in ``option_docs``. + options: dict[str, Any] = Field(default_factory=dict) + + @field_validator("endpoint") + @classmethod + def _https_only(cls, v: str) -> str: + v = v.strip().rstrip("/") + if not v.lower().startswith("https://"): + raise ValueError("Array endpoint must be https://") + return v + + +@dataclass(frozen=True) +class Capabilities: + protocols: frozenset[Protocol] + #: Volume sizes are rounded up to a multiple of this. + size_granularity_bytes: int = 512 + min_size_bytes: int = 1 << 20 + #: Longest volume / host name the array accepts. + max_name_len: int = 63 + #: Whether resize can only grow (true for every supported array today). + grow_only: bool = True + #: Most initiators one host may have; ``None`` = unlimited. PowerFlex SDC + #: hosts are exactly one client, so 1 there. + max_initiators_per_host: int | None = None + + +@dataclass +class OptionDoc: + name: str + help: str + required: bool = False + default: Any = None + + +_TRUE = frozenset({"true", "yes", "on", "1"}) +_FALSE = frozenset({"false", "no", "off", "0"}) + + +def _coerce_option(owner: str, name: str, value: Any, default: Any) -> Any: + if not isinstance(value, str): + return value + text = value.strip() + if text.startswith("{"): + try: + return json.loads(text) + except ValueError: + raise ArrayError(f"{owner} option {name!r} is not valid JSON") from None + if isinstance(default, bool): + if text.lower() in _TRUE: + return True + if text.lower() in _FALSE: + return False + raise ArrayError(f"{owner} option {name!r} must be true or false, got {value!r}") + if isinstance(default, int): + try: + return int(text) + except ValueError: + raise ArrayError(f"{owner} option {name!r} must be a whole number, got {value!r}") from None + if isinstance(default, float): + try: + return float(text) + except ValueError: + raise ArrayError(f"{owner} option {name!r} must be a number, got {value!r}") from None + if isinstance(default, (list, tuple)): + return [item for item in (part.strip() for part in text.replace(",", " ").split()) if item] + return value + + +@dataclass +class AdapterContext: + """What an adapter is constructed with.""" + + config: ArrayConfig + credential: Credential + session: Any # simplyblock_core.storage_arrays.http.RestSession + extra: dict[str, Any] = field(default_factory=dict) + + +class ArrayAdapter(ABC): + """One vendor's implementation of the contract. + + Rules every adapter follows (the contract suite checks them): + + * ``create_volume`` receives a name that is already unique and short + enough, and a size already rounded to ``capabilities``. + * ``get_volume``/``find_volume`` return ``None`` for a missing volume; + every other method raises :class:`ArrayNotFoundError` for one. + * ``ensure_host`` is idempotent: an existing host of that name gets any + missing initiators added; an initiator registered to a *different* host + raises :class:`ArrayConflictError`. + * ``map_volume`` is idempotent and returns the existing mapping if there is + one; ``unmap_volume`` of an absent mapping is a no-op. + * ``delete_volume`` is only called with no mappings left. + * Errors never include credential values. + """ + + vendor: ClassVar[str] + display_name: ClassVar[str] + capabilities: ClassVar[Capabilities] + #: Accepted credential shapes, as alternatives of required field names, + #: e.g. ``({"api_token"}, {"username", "password"})``. + credential_fields: ClassVar[tuple[frozenset[str], ...]] = ( + frozenset({"username", "password"}), + ) + option_docs: ClassVar[tuple[OptionDoc, ...]] = () + + def __init__(self, ctx: AdapterContext): + self.config = ctx.config + self.credential = ctx.credential + self.session = ctx.session + self.options: MappingABC[str, Any] = ctx.config.options + self._check_credential_shape() + self._check_options() + + # -- helpers ------------------------------------------------------------ + + def _check_credential_shape(self) -> None: + have = self.credential.field_names + if not any(shape <= have for shape in self.credential_fields): + wanted = " or ".join("+".join(sorted(s)) for s in self.credential_fields) + raise ArrayAuthError( + f"{self.display_name} needs credential fields {wanted}; " + f"got {', '.join(sorted(have)) or 'none'}") + + def _check_options(self) -> None: + missing = [o.name for o in self.option_docs if o.required and o.name not in self.options] + if missing: + raise ArrayError(f"{self.display_name} array {self.config.id} needs options: " + + ", ".join(missing)) + known = {o.name for o in self.option_docs} + unknown = sorted(set(self.options) - known) + if unknown: + raise ArrayError(f"Unknown {self.display_name} options: {', '.join(unknown)}") + for o in self.option_docs: + if o.name in self.options: + self.option(o.name) # a value that cannot be read fails here, at registration + + def option(self, name: str) -> Any: + """The option's value, or its default. A string (options set through + Kubernetes are strings) is read as the type of the default: a boolean + from true/false/yes/no/on/off/1/0, a number, a comma-separated list, + and a JSON object where it starts with ``{``.""" + doc = next((o for o in self.option_docs if o.name == name), None) + if name in self.options: + return _coerce_option(self.display_name, name, self.options[name], + doc.default if doc else None) + if doc is not None: + return doc.default + raise KeyError(name) + + def secret(self, name: str) -> str: + return self.credential.get(name).get_secret_value() + + def has_secret(self, name: str) -> bool: + return name in self.credential.field_names + + # -- lifecycle ---------------------------------------------------------- + + def login(self) -> None: + """Establish a session. Called lazily by the client and again after + an :class:`ArrayAuthError`. Basic-auth adapters may leave this empty.""" + + def close(self) -> None: + """Release the session (best effort, never raises).""" + + # -- contract ----------------------------------------------------------- + + @abstractmethod + def get_info(self) -> ArrayInfo: ... + + @abstractmethod + def create_volume(self, name: str, size_bytes: int) -> Volume: ... + + @abstractmethod + def get_volume(self, volume_id: str) -> Volume | None: ... + + @abstractmethod + def find_volume(self, name: str) -> Volume | None: ... + + @abstractmethod + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: ... + + @abstractmethod + def delete_volume(self, volume_id: str) -> None: ... + + @abstractmethod + def ensure_host(self, spec: HostSpec) -> str: + """Return the host id.""" + + @abstractmethod + def delete_host(self, host_id: str) -> None: + """Delete a host that has no mappings left (no-op if absent).""" + + @abstractmethod + def map_volume(self, volume_id: str, host_id: str) -> Mapping: ... + + @abstractmethod + def unmap_volume(self, volume_id: str, host_id: str) -> None: ... + + @abstractmethod + def list_mappings(self, volume_id: str) -> list[Mapping]: ... diff --git a/simplyblock_core/storage_arrays/client.py b/simplyblock_core/storage_arrays/client.py new file mode 100644 index 0000000000..cccef18751 --- /dev/null +++ b/simplyblock_core/storage_arrays/client.py @@ -0,0 +1,288 @@ +"""Vendor-neutral front end for array adapters. + +:class:`ArrayClient` is what the rest of simplyblock calls. It + +* resolves the array's credential through the cluster's + :class:`~simplyblock_core.credentials.CredentialProvider`, bound to the + array's id and endpoint (so a rewritten endpoint cannot receive it); +* builds the adapter lazily and, when the array rejects the session or the + credential, invalidates the cached credential, logs in again and retries + the operation once (this is how rotated secrets are picked up); +* names volumes ``sb-`` from the simplyblock volume UUID, so names are + unique, fit every array's length limit, and mark ownership; +* rounds sizes up to the array's granularity; +* makes create/map/unmap/delete idempotent so an interrupted task can be + re-run; +* refuses to resize, unmap or delete a volume that is not simplyblock-owned + (name prefix) or whose WWN/NGUID differs from the one simplyblock recorded. +""" +from __future__ import annotations + +import logging +import threading +import uuid as uuid_mod +from collections.abc import Callable +from typing import Any, Self, TypeVar +from urllib.parse import urlsplit + +from requests.adapters import BaseAdapter + +from simplyblock_core.credentials import ( + CREDENTIAL_REF_ADAPTER, + Credential, + CredentialProvider, +) + +from .base import ( + AdapterContext, + ArrayAdapter, + ArrayAuthError, + ArrayConfig, + ArrayConflictError, + ArrayError, + ArrayInfo, + HostSpec, + Mapping, + Volume, +) +from .http import RestSession +from .registry import adapter_class, canonical_vendor + +logger = logging.getLogger(__name__) + +MANAGED_PREFIX = "sb-" +_NAME_HEX_LEN = 24 # 96 bits of the volume UUID: ample, and fits 31-char limits + +T = TypeVar("T") + + +def credential_context(config: ArrayConfig) -> str: + """The context an array's credential is bound to: id + scheme/host/port.""" + parts = urlsplit(config.endpoint) + port = parts.port or 443 + return f"array/{config.id}/{parts.scheme}://{(parts.hostname or '').lower()}:{port}" + + +def managed_name(volume_uuid: str, max_len: int, prefix: str = MANAGED_PREFIX) -> str: + hex_id = uuid_mod.UUID(str(volume_uuid)).hex + return (prefix + hex_id[:_NAME_HEX_LEN])[:max_len] + + +def round_size(size_bytes: int, granularity: int, minimum: int) -> int: + size = max(int(size_bytes), minimum) + return -(-size // granularity) * granularity + + +class ArrayClient: + """Thread-safe: operations on one array are serialized.""" + + def __init__( + self, + config: ArrayConfig, + credentials: CredentialProvider, + *, + transport: BaseAdapter | None = None, + session_kwargs: dict[str, Any] | None = None, + ): + self.config = config + self.vendor = canonical_vendor(config.vendor) + self.adapter_cls = adapter_class(self.vendor) + self._credentials = credentials + self._ref = CREDENTIAL_REF_ADAPTER.validate_python(config.credential_ref) + self._context = credential_context(config) + self._transport = transport + self._session_kwargs = session_kwargs or {} + self._adapter: ArrayAdapter | None = None + self._lock = threading.RLock() + + @property + def capabilities(self): + return self.adapter_cls.capabilities + + # -- adapter lifecycle ---------------------------------------------------- + + def _build(self) -> ArrayAdapter: + credential: Credential = self._credentials.resolve(self._ref, self._context) + session = RestSession.for_config(self.config, transport=self._transport, + **self._session_kwargs) + adapter = self.adapter_cls(AdapterContext(self.config, credential, session)) + try: + adapter.login() + except BaseException: + adapter.close() + session.close() + raise + return adapter + + def _drop(self) -> None: + if self._adapter is not None: + try: + self._adapter.close() + except Exception: # pragma: no cover - best effort + logger.debug("adapter close failed", exc_info=True) + try: + self._adapter.session.close() + except Exception: # pragma: no cover + pass + self._adapter = None + + def close(self) -> None: + with self._lock: + self._drop() + + def __enter__(self) -> Self: + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + def _call(self, op: str, fn: Callable[[ArrayAdapter], T]) -> T: + with self._lock: + for attempt in (1, 2): + try: + if self._adapter is None: + self._adapter = self._build() + return fn(self._adapter) + except ArrayAuthError: + self._drop() + if attempt == 2: + raise + logger.info("Array %s rejected the session during %s; re-authenticating", + self.config.id, op) + self._credentials.invalidate(self._ref) + raise AssertionError("unreachable") # pragma: no cover + + # -- checks ----------------------------------------------------------------- + + def _owned(self, vol: Volume, expected_identifier: str | None, action: str, + volume_uuid: str | None = None) -> None: + """Refuse to touch a volume simplyblock did not create, or that is not + the one it recorded. The name check matters where the array reuses + identities: an LDEV number / volume id (and the WWN derived from it) + can come back after a delete, the name derived from our UUID cannot.""" + if not vol.name.startswith(MANAGED_PREFIX): + raise ArrayConflictError( + f"Refusing to {action} volume {vol.name!r} on array {self.config.id}: " + "not created by simplyblock") + if volume_uuid is not None and vol.name != self.volume_name(volume_uuid): + raise ArrayConflictError( + f"Refusing to {action} volume {vol.name!r} on array {self.config.id}: " + f"name does not match recorded volume {volume_uuid}") + if expected_identifier is not None and vol.identifier.lower() != expected_identifier.lower(): + raise ArrayConflictError( + f"Refusing to {action} volume {vol.name!r} on array {self.config.id}: " + f"identifier {vol.identifier} does not match recorded {expected_identifier}") + + # -- operations ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + return self._call("get_info", lambda a: a.get_info()) + + def volume_name(self, volume_uuid: str) -> str: + return managed_name(volume_uuid, self.capabilities.max_name_len) + + def create_volume(self, volume_uuid: str, size_bytes: int) -> Volume: + """Create (or, on re-run, return) the array volume for a simplyblock + volume. An existing volume of that name smaller than requested is grown.""" + name = self.volume_name(volume_uuid) + caps = self.capabilities + size = round_size(size_bytes, caps.size_granularity_bytes, caps.min_size_bytes) + + def op(a: ArrayAdapter) -> Volume: + existing = a.find_volume(name) + if existing is not None: + if existing.size_bytes < size: + return a.resize_volume(existing.id, size) + return existing + vol = a.create_volume(name, size) + logger.info("Array %s: created volume %s (%s, %d bytes)", + self.config.id, vol.name, vol.identifier, vol.size_bytes) + return vol + + return self._call("create_volume", op) + + def get_volume(self, volume_id: str) -> Volume | None: + return self._call("get_volume", lambda a: a.get_volume(volume_id)) + + def find_volume(self, volume_uuid: str) -> Volume | None: + name = self.volume_name(volume_uuid) + return self._call("find_volume", lambda a: a.find_volume(name)) + + def resize_volume(self, volume_id: str, size_bytes: int, + expected_identifier: str | None = None, + volume_uuid: str | None = None) -> Volume: + caps = self.capabilities + size = round_size(size_bytes, caps.size_granularity_bytes, caps.min_size_bytes) + + def op(a: ArrayAdapter) -> Volume: + vol = a.get_volume(volume_id) + if vol is None: + raise ArrayError(f"Volume {volume_id} not found on array {self.config.id}") + self._owned(vol, expected_identifier, "resize", volume_uuid) + if size == vol.size_bytes: + return vol + if size < vol.size_bytes and caps.grow_only: + raise ArrayConflictError( + f"Refusing to shrink volume {vol.name} from {vol.size_bytes} to {size} bytes") + new = a.resize_volume(vol.id, size) + logger.info("Array %s: resized volume %s %d -> %d bytes", + self.config.id, vol.name, vol.size_bytes, new.size_bytes) + return new + + return self._call("resize_volume", op) + + def delete_volume(self, volume_id: str, expected_identifier: str | None = None, + volume_uuid: str | None = None) -> bool: + """Unmap from every host and delete. Returns False if already gone. + + Callers should pass both ``expected_identifier`` and ``volume_uuid`` + from their record; together they pin the exact volume.""" + def op(a: ArrayAdapter) -> bool: + vol = a.get_volume(volume_id) + if vol is None: + return False + self._owned(vol, expected_identifier, "delete", volume_uuid) + for m in a.list_mappings(vol.id): + a.unmap_volume(vol.id, m.host_id) + a.delete_volume(vol.id) + logger.info("Array %s: deleted volume %s (%s)", self.config.id, vol.name, vol.identifier) + return True + + return self._call("delete_volume", op) + + def ensure_host(self, spec: HostSpec) -> str: + unsupported = spec.protocols - self.capabilities.protocols + if unsupported: + raise ArrayError( + f"{self.adapter_cls.display_name} adapter does not support " + + ", ".join(sorted(p.value for p in unsupported))) + limit = self.capabilities.max_initiators_per_host + if limit is not None and len(spec.initiators) > limit: + raise ArrayError( + f"{self.adapter_cls.display_name} hosts take at most {limit} initiator(s); " + f"got {len(spec.initiators)}") + if len(spec.name) > self.capabilities.max_name_len: + raise ArrayError(f"Host name {spec.name!r} exceeds " + f"{self.capabilities.max_name_len} characters") + return self._call("ensure_host", lambda a: a.ensure_host(spec)) + + def delete_host(self, host_id: str) -> None: + self._call("delete_host", lambda a: a.delete_host(host_id)) + + def map_volume(self, volume_id: str, host_id: str, + expected_identifier: str | None = None, + volume_uuid: str | None = None) -> Mapping: + def op(a: ArrayAdapter) -> Mapping: + vol = a.get_volume(volume_id) + if vol is None: + raise ArrayError(f"Volume {volume_id} not found on array {self.config.id}") + self._owned(vol, expected_identifier, "map", volume_uuid) + return a.map_volume(vol.id, host_id) + + return self._call("map_volume", op) + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + self._call("unmap_volume", lambda a: a.unmap_volume(volume_id, host_id)) + + def list_mappings(self, volume_id: str) -> list[Mapping]: + return self._call("list_mappings", lambda a: a.list_mappings(volume_id)) diff --git a/simplyblock_core/storage_arrays/http.py b/simplyblock_core/storage_arrays/http.py new file mode 100644 index 0000000000..0f4e0819d9 --- /dev/null +++ b/simplyblock_core/storage_arrays/http.py @@ -0,0 +1,200 @@ +"""Small HTTPS/JSON session shared by all array adapters. + +Adapters never build ``requests`` sessions themselves, so TLS verification, +timeouts, retries and error mapping are uniform, and tests can swap the +transport for an in-process fake (see ``storage_arrays.testing``). +""" +from __future__ import annotations + +import logging +import os +import tempfile +import time +from collections.abc import Callable, Iterable +from typing import Any + +import requests +from requests.adapters import BaseAdapter + +from .base import ( + ArrayAuthError, + ArrayConfig, + ArrayConflictError, + ArrayNotFoundError, + ArrayRequestError, +) + +logger = logging.getLogger(__name__) + +_IDEMPOTENT = frozenset({"GET", "HEAD", "OPTIONS"}) +_RETRY_STATUSES = frozenset({502, 503, 504}) +_MAX_ERROR_BODY = 600 + + +class RestSession: + """``requests`` wrapper bound to one array endpoint. + + * ``verify``: ``True`` (system CAs), a CA bundle path, or ``False``. + * Non-2xx responses raise: 401/403 -> :class:`ArrayAuthError`, 404 -> + :class:`ArrayNotFoundError`, 409 -> :class:`ArrayConflictError`, other -> + :class:`ArrayRequestError`. Pass ``ok=`` to accept extra statuses. + * Idempotent requests are retried on connection errors and 502/503/504. + """ + + def __init__( + self, + base_url: str, + *, + verify: bool | str = True, + timeout_s: float = 60.0, + retries: int = 2, + backoff_s: float = 0.5, + transport: BaseAdapter | None = None, + sleep: Callable[[float], None] = time.sleep, + ): + self.base_url = base_url.rstrip("/") + self.timeout_s = timeout_s + self.retries = retries + self.backoff_s = backoff_s + #: Used for retry backoff and by adapters that poll async jobs, so + #: tests can make all waiting instant. + self.sleep = sleep + self.http = requests.Session() + self.http.verify = verify + self.http.headers.update({"Accept": "application/json"}) + if transport is not None: + self.http.mount("https://", transport) + self.http.mount("http://", transport) + self._ca_file: str | None = None + + @classmethod + def for_config(cls, config: ArrayConfig, *, transport: BaseAdapter | None = None, + base_url: str | None = None, **kw: Any) -> RestSession: + verify: bool | str = config.verify_tls + ca_file = None + if config.verify_tls and config.ca_bundle: + fd, ca_file = tempfile.mkstemp(prefix="sb-array-ca-", suffix=".pem") + with os.fdopen(fd, "w") as f: + f.write(config.ca_bundle) + verify = ca_file + session = cls(base_url or config.endpoint, verify=verify, timeout_s=config.timeout_s, + transport=transport, **kw) + session._ca_file = ca_file + return session + + # -- auth helpers --------------------------------------------------------- + + @property + def headers(self) -> Any: + return self.http.headers + + def set_basic_auth(self, username: str, password: str) -> None: + self.http.auth = (username, password) + + def clear_auth(self) -> None: + self.http.auth = None + + # -- requests ------------------------------------------------------------- + + def url(self, path: str) -> str: + if path.startswith(("http://", "https://")): + return path + return f"{self.base_url}/{path.lstrip('/')}" + + def request( + self, + method: str, + path: str, + *, + params: Any = None, + json: Any = None, + data: Any = None, + headers: dict[str, str] | None = None, + ok: Iterable[int] = (), + auth: Any = None, + ) -> requests.Response: + method = method.upper() + attempts = 1 + (self.retries if method in _IDEMPOTENT else 0) + ok_extra = frozenset(ok) + last_exc: Exception | None = None + for attempt in range(attempts): + try: + kwargs: dict[str, Any] = dict(params=params, json=json, data=data, + headers=headers, timeout=self.timeout_s) + if auth is not None: + kwargs["auth"] = auth + resp = self.http.request(method, self.url(path), **kwargs) + except requests.RequestException as e: + last_exc = e + if attempt + 1 < attempts: + self.sleep(self.backoff_s * (attempt + 1)) + continue + raise ArrayRequestError( + f"{method} {path}: {type(e).__name__}: {_safe(str(e))}") from e + if resp.status_code in _RETRY_STATUSES and attempt + 1 < attempts: + self.sleep(self.backoff_s * (attempt + 1)) + continue + if 200 <= resp.status_code < 300 or resp.status_code in ok_extra: + return resp + raise error_for(method, path, resp) + raise ArrayRequestError(f"{method} {path}: giving up") from last_exc # pragma: no cover + + def get(self, path: str, **kw: Any) -> requests.Response: + return self.request("GET", path, **kw) + + def post(self, path: str, **kw: Any) -> requests.Response: + return self.request("POST", path, **kw) + + def put(self, path: str, **kw: Any) -> requests.Response: + return self.request("PUT", path, **kw) + + def patch(self, path: str, **kw: Any) -> requests.Response: + return self.request("PATCH", path, **kw) + + def delete(self, path: str, **kw: Any) -> requests.Response: + return self.request("DELETE", path, **kw) + + def close(self) -> None: + try: + self.http.close() + finally: + if self._ca_file: + try: + os.unlink(self._ca_file) + except OSError: + pass + self._ca_file = None + + +def _safe(text: str) -> str: + return text[:_MAX_ERROR_BODY] + + +def body_text(resp: requests.Response) -> str: + try: + return _safe(resp.text or "") + except Exception: # pragma: no cover - decoding issues + return "" + + +def error_for(method: str, path: str, resp: requests.Response) -> Exception: + msg = f"{method} {path} -> HTTP {resp.status_code}: {body_text(resp)}" + if resp.status_code in (401, 403): + return ArrayAuthError(msg) + if resp.status_code == 404: + return ArrayNotFoundError(msg) + if resp.status_code == 409: + return ArrayConflictError(msg) + return ArrayRequestError(msg, status=resp.status_code) + + +def json_of(resp: requests.Response) -> Any: + """Response body as JSON (``None`` for an empty body).""" + if not resp.content: + return None + try: + return resp.json() + except ValueError as e: + raise ArrayRequestError( + f"Expected JSON from {resp.request.method} {resp.url}, got: {body_text(resp)}", + status=resp.status_code) from e diff --git a/simplyblock_core/storage_arrays/registry.py b/simplyblock_core/storage_arrays/registry.py new file mode 100644 index 0000000000..b868b75571 --- /dev/null +++ b/simplyblock_core/storage_arrays/registry.py @@ -0,0 +1,61 @@ +"""Vendor key -> adapter class. Adapters are imported lazily, on first use.""" +from __future__ import annotations + +import importlib + +from .base import ArrayAdapter, ArrayUnsupportedError + +#: vendor key -> "module:Class" (module relative to this package) +ADAPTERS: dict[str, str] = { + "pure-flasharray": "adapters.pure:PureFlashArrayAdapter", + "netapp-ontap": "adapters.netapp_ontap:NetAppOntapAdapter", + "dell-powerstore": "adapters.dell_powerstore:DellPowerStoreAdapter", + "dell-powermax": "adapters.dell_powermax:DellPowerMaxAdapter", + "dell-powerflex": "adapters.dell_powerflex:DellPowerFlexAdapter", + "hpe-wsapi": "adapters.hpe_wsapi:HpeWsapiAdapter", + "hpe-nimble": "adapters.hpe_nimble:HpeNimbleAdapter", + "ibm-flashsystem": "adapters.ibm_flashsystem:IbmFlashSystemAdapter", + "hitachi-vsp": "adapters.hitachi_vsp:HitachiVspAdapter", + "huawei-dorado": "adapters.huawei_dorado:HuaweiDoradoAdapter", + "infinidat-infinibox": "adapters.infinidat:InfinidatAdapter", +} + +#: Other names customers use for the same API. +ALIASES: dict[str, str] = { + "pure": "pure-flasharray", + "netapp": "netapp-ontap", + "netapp-asa": "netapp-ontap", + "lenovo-thinksystem-dm": "netapp-ontap", + "powerstore": "dell-powerstore", + "powermax": "dell-powermax", + "powerflex": "dell-powerflex", + "hpe-alletra-mp": "hpe-wsapi", + "hpe-primera": "hpe-wsapi", + "hpe-3par": "hpe-wsapi", + "hpe-alletra-6000": "hpe-nimble", + "hpe-alletra-5000": "hpe-nimble", + "ibm-storage-virtualize": "ibm-flashsystem", + "ibm-svc": "ibm-flashsystem", + "hitachi": "hitachi-vsp", + "huawei-oceanstor": "huawei-dorado", + "infinidat": "infinidat-infinibox", +} + + +def canonical_vendor(vendor: str) -> str: + key = vendor.strip().lower() + key = ALIASES.get(key, key) + if key not in ADAPTERS: + raise ArrayUnsupportedError( + f"Unknown array vendor {vendor!r}; supported: {', '.join(sorted(ADAPTERS))}") + return key + + +def adapter_class(vendor: str) -> type[ArrayAdapter]: + target = ADAPTERS[canonical_vendor(vendor)] + module_name, _, class_name = target.partition(":") + module = importlib.import_module(f"{__package__}.{module_name}") + cls = getattr(module, class_name) + if not (isinstance(cls, type) and issubclass(cls, ArrayAdapter)): + raise TypeError(f"{target} is not an ArrayAdapter") + return cls diff --git a/simplyblock_core/storage_arrays/testing/__init__.py b/simplyblock_core/storage_arrays/testing/__init__.py new file mode 100644 index 0000000000..35f5276847 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/__init__.py @@ -0,0 +1,352 @@ +"""Stateful fakes of each vendor's management REST API. + +Each ``testing/.py`` module provides + +* a :class:`FakeArray` subclass that keeps volumes/hosts/mappings in memory + and answers the subset of the vendor API its adapter uses, with the + vendor's status codes, error bodies and quirks (async jobs, session + tokens, size units); and +* ``contract_setup()`` returning a :class:`ContractSetup` for the shared + contract suite. + +Fakes plug into ``requests`` in-process through :class:`FakeTransport` (no +sockets), or run as a real HTTPS server with :func:`serve` / ``python -m +simplyblock_core.storage_arrays.testing`` for end-to-end and UI testing. + +A fake encodes our reading of the vendor API. Passing the contract suite +proves the adapter logic; it does not prove API fidelity -- that needs the +same suite against a vendor simulator or a real array (``live`` tier). +""" +from __future__ import annotations + +import base64 +import http.client +import io +import json +import logging +import re +import threading +import types +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import parse_qs, urlsplit + +import requests +from requests.adapters import BaseAdapter +from requests.structures import CaseInsensitiveDict + +from simplyblock_core.credentials import Credential + +from ..base import ArrayConfig, Initiator, Protocol + +Handler = Callable[..., "Reply"] + + +@dataclass +class Request: + method: str + path: str + query: dict[str, list[str]] + headers: CaseInsensitiveDict + body: bytes + + def json(self) -> Any: + return json.loads(self.body) if self.body else None + + def q(self, name: str, default: str | None = None) -> str | None: + values = self.query.get(name) + return values[0] if values else default + + def basic_auth(self) -> tuple[str, str] | None: + header = self.headers.get("Authorization", "") + if not header.lower().startswith("basic "): + return None + user, _, pw = base64.b64decode(header[6:]).decode().partition(":") + return user, pw + + +@dataclass +class Reply: + status: int = 200 + body: Any = None + headers: dict[str, str] = field(default_factory=dict) + + +def route(method: str, pattern: str): + """Decorate a FakeArray method as the handler of ``method`` + path regex. + Named groups are passed as keyword arguments.""" + def deco(fn): + fn._route = (method.upper(), re.compile(f"^{pattern}$")) + return fn + return deco + + +class FakeArray: + """Base class: routing, fault injection, request log, thread safety.""" + + vendor: str = "" + + def __init__(self) -> None: + self.lock = threading.RLock() + self.requests: list[tuple[str, str]] = [] + self._faults: list[tuple[re.Pattern[str] | None, Reply, int]] = [] + self._routes: list[tuple[str, re.Pattern[str], Handler]] = [] + for name in dir(type(self)): + fn = getattr(type(self), name) + r = getattr(fn, "_route", None) + if r: + self._routes.append((r[0], r[1], getattr(self, name))) + self._id = 0 + + # -- helpers for subclasses ---------------------------------------------- + + def next_id(self) -> int: + self._id += 1 + return self._id + + # -- fault injection -------------------------------------------------------- + + def fail_next(self, status: int, body: Any = None, *, path: str | None = None, + times: int = 1) -> None: + """Answer the next ``times`` requests (optionally only those whose path + matches ``path`` regex) with ``status`` and ``body``.""" + with self.lock: + self._faults.append((re.compile(path) if path else None, Reply(status, body), times)) + + def expire_sessions(self) -> None: + """Invalidate every session token, as an array restart or timeout would. + Basic-auth fakes have nothing to expire.""" + + def set_credential(self, **fields: str) -> None: + """Change the credential the fake accepts (simulates rotation).""" + raise NotImplementedError + + # -- dispatch --------------------------------------------------------------- + + def handle(self, req: Request) -> Reply: + with self.lock: + self.requests.append((req.method, req.path)) + for i, (pattern, reply, times) in enumerate(self._faults): + if pattern is None or pattern.search(req.path): + if times <= 1: + del self._faults[i] + else: + self._faults[i] = (pattern, reply, times - 1) + return reply + for method, pattern, handler in self._routes: + if method != req.method: + continue + m = pattern.match(req.path) + if m: + try: + return handler(req, **m.groupdict()) + except FakeError as e: + return e.reply + return self.not_routed(req) + + def not_routed(self, req: Request) -> Reply: + return Reply(404, {"error": f"fake: no route for {req.method} {req.path}"}) + + +# The LIO data path (testing.lio) reads a fake through two accessors, which +# every vendor fake implements: +# +# * ``lio_volume_ids()``: the ids of the live volumes, as the vendor's adapter +# reports them in ``Volume.id``; +# * ``lio_host_iqns(host_id)``: the iSCSI initiator names of the host the +# adapter knows as ``host_id`` (the id ``ensure_host`` returned or the name +# ``list_mappings`` reports), lower case; +# * ``lio_all_iqns()``: the iSCSI initiator names of every host, so that a +# registered host can log in before anything is mapped to it. + +class FakeError(Exception): + """Raise inside a handler to answer with ``reply``.""" + + def __init__(self, status: int, body: Any = None, headers: dict[str, str] | None = None): + super().__init__(status) + self.reply = Reply(status, body, headers or {}) + + +class FakeTransport(BaseAdapter): + """A ``requests`` transport adapter answering from a :class:`FakeArray`.""" + + def __init__(self, fake: FakeArray): + super().__init__() + self.fake = fake + + def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None): + parts = urlsplit(request.url) + body = request.body or b"" + if isinstance(body, str): + body = body.encode() + req = Request( + method=request.method.upper(), + path=parts.path, + query=parse_qs(parts.query, keep_blank_values=True), + headers=CaseInsensitiveDict(request.headers), + body=body, + ) + reply = self.fake.handle(req) + return _to_response(request, reply) + + def close(self): + pass + + +def _encode(reply: Reply) -> tuple[bytes, dict[str, str]]: + headers = dict(reply.headers) + if reply.body is None: + return b"", headers + if isinstance(reply.body, bytes): + return reply.body, headers + if isinstance(reply.body, str): + headers.setdefault("Content-Type", "text/plain") + return reply.body.encode(), headers + headers.setdefault("Content-Type", "application/json") + return json.dumps(reply.body).encode(), headers + + +class _Raw(io.BytesIO): + """Body stream that also carries the headers the way urllib3 does, so + ``requests`` stores Set-Cookie values in the session's cookie jar exactly + as it would for a real response.""" + + def __init__(self, content: bytes, headers: dict[str, str]): + super().__init__(content) + msg = http.client.HTTPMessage() + for k, v in headers.items(): + msg[k] = v + self._original_response = types.SimpleNamespace(msg=msg) + + +def _to_response(request, reply: Reply) -> requests.Response: + content, headers = _encode(reply) + resp = requests.Response() + resp.status_code = reply.status + resp.headers = CaseInsensitiveDict(headers) + resp.raw = _Raw(content, headers) + resp._content = content + resp.url = request.url + resp.request = request + resp.encoding = "utf-8" + resp.reason = "" + return resp + + +@dataclass +class ContractSetup: + """What the shared contract suite needs from a vendor fake.""" + + fake: FakeArray + config: ArrayConfig + credential: Credential + #: Credential the fake rejects. + bad_credential: Credential + #: Two distinct initiator sets per supported protocol, for two hosts. + initiators: dict[Protocol, tuple[tuple[Initiator, ...], tuple[Initiator, ...]]] + #: Create a volume directly on the fake (bypassing the client), for the + #: "not simplyblock-owned" checks. Returns the array's volume id. + create_foreign_volume: Callable[[str, int], str] + #: Whether mapping yields a LUN/NSID number. + has_lun_numbers: bool = True + + +def make_config(vendor: str, endpoint: str, **options: Any) -> ArrayConfig: + return ArrayConfig( + id=f"fake-{vendor}", + vendor=vendor, + endpoint=endpoint, + credential_ref={"kind": "vault-kv", "mount": "m", "path": f"arrays/{vendor}"}, + options=options, + ) + + +# -- standalone HTTPS mock server ------------------------------------------------ + + +def serve(fake: FakeArray, host: str = "127.0.0.1", port: int = 8443, + certfile: str | None = None, keyfile: str | None = None, + after_request: Callable[[], None] | None = None): + """Serve ``fake`` over HTTPS (a throwaway self-signed certificate is + generated unless ``certfile``/``keyfile`` are given). Returns the server; + call ``serve_forever()`` on it. ``after_request`` runs after every + request, before the answer is sent: the LIO data path (``testing.lio``) + catches up there, so a mapping the caller was told about is already + presented. GETs included: on arrays with asynchronous jobs, a change + takes effect while its job is polled.""" + import ssl + import tempfile + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def _do(self): + parts = urlsplit(self.path) + length = int(self.headers.get("Content-Length") or 0) + body = self.rfile.read(length) if length else b"" + reply = fake.handle(Request( + method=self.command, path=parts.path, + query=parse_qs(parts.query, keep_blank_values=True), + headers=CaseInsensitiveDict(dict(self.headers.items())), body=body)) + if after_request is not None: + try: + after_request() + except Exception: # noqa: BLE001 - the API answer still goes out + logging.getLogger(__name__).exception("after_request failed") + content, headers = _encode(reply) + self.send_response(reply.status) + for k, v in headers.items(): + self.send_header(k, v) + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + + do_GET = do_POST = do_PUT = do_PATCH = do_DELETE = _do + + def log_message(self, fmt, *args): # quieter default logging + pass + + server = ThreadingHTTPServer((host, port), Handler) + if certfile is None: + tmp = tempfile.mkdtemp(prefix="sb-array-mock-") + certfile, keyfile = f"{tmp}/cert.pem", f"{tmp}/key.pem" + _self_signed(host, certfile, keyfile) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + ctx.load_cert_chain(certfile, keyfile) + server.socket = ctx.wrap_socket(server.socket, server_side=True) + server.certfile = certfile # type: ignore[attr-defined] + return server + + +def _self_signed(host: str, certfile: str, keyfile: str) -> None: + import datetime + import ipaddress + + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import ec + from cryptography.x509.oid import NameOID + + key = ec.generate_private_key(ec.SECP256R1()) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, host)]) + try: + san: x509.GeneralName = x509.IPAddress(ipaddress.ip_address(host)) + except ValueError: + san = x509.DNSName(host) + now = datetime.datetime.now(datetime.UTC) + cert = (x509.CertificateBuilder().subject_name(name).issuer_name(name) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(minutes=5)) + .not_valid_after(now + datetime.timedelta(days=30)) + .add_extension(x509.SubjectAlternativeName([san]), critical=False) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .sign(key, hashes.SHA256())) + with open(certfile, "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) + with open(keyfile, "wb") as f: + f.write(key.private_bytes(serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption())) diff --git a/simplyblock_core/storage_arrays/testing/__main__.py b/simplyblock_core/storage_arrays/testing/__main__.py new file mode 100644 index 0000000000..95ce4009ce --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/__main__.py @@ -0,0 +1,77 @@ +"""Run a vendor fake as an HTTPS mock server. + + python -m simplyblock_core.storage_arrays.testing --vendor pure-flasharray --port 8443 + +Prints the accepted credential, the CA certificate path (pass its content as +the array's ``ca_bundle``) and the options to register the array with. + +With ``--lio`` (root, LIO modules and python3-rtslib-fb on the host) the +fake's volumes are real iSCSI LUNs of the kernel target, presented to the +hosts the fake maps them to (see ``testing.lio``):: + + python -m simplyblock_core.storage_arrays.testing --vendor pure-flasharray \ + --host 0.0.0.0 --lio --lio-portal 10.0.0.5:3260 \ + --lio-allow iqn.1994-05.com.redhat:node1,iqn.1994-05.com.redhat:node2 +""" +import argparse +import importlib +import json + +from ..registry import ADAPTERS, canonical_vendor + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--vendor", required=True, help=", ".join(sorted(ADAPTERS))) + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8443) + parser.add_argument("--lio", action="store_true", + help="present the volumes as iSCSI LUNs of the kernel target") + parser.add_argument("--lio-target", default="iqn.2026-11.io.simplyblock:mock-array", + help="IQN of the iSCSI target") + parser.add_argument("--lio-portal", default="0.0.0.0:3260", help="portal address:port") + parser.add_argument("--lio-dir", default="/var/lib/sb-mock-lio", help="where the LUN files live") + parser.add_argument("--lio-allow", default="", + help="comma-separated initiators allowed to log in before they are registered") + args = parser.parse_args() + + vendor = canonical_vendor(args.vendor) + module = importlib.import_module( + f"{__package__}.{ADAPTERS[vendor].split(':')[0].split('.')[-1]}") + setup = module.contract_setup() + mirror = None + if args.lio: + from . import FakeTransport + from .lio import ConfigfsLio, LioMirror + ip, _, port = args.lio_portal.rpartition(":") + backend = ConfigfsLio(args.lio_target, args.lio_dir, (ip or "0.0.0.0", int(port))) + mirror = LioMirror(setup.fake, setup.config, setup.credential, backend, + transport=FakeTransport(setup.fake), + allow=[i for i in args.lio_allow.split(",") if i]) + mirror.reconcile() + server = serve_fake(setup.fake, args.host, args.port, + mirror.reconcile if mirror is not None else None) + print(json.dumps({ + "vendor": vendor, + "endpoint": f"https://{args.host}:{args.port}", + "ca_bundle_file": server.certfile, + "credential_fields": sorted(setup.credential.field_names), + "credential": {k: setup.credential.get(k).get_secret_value() + for k in setup.credential.field_names}, + "options": setup.config.options, + "iscsi_target": args.lio_target if args.lio else None, + "iscsi_portal": args.lio_portal if args.lio else None, + }, indent=2), flush=True) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + + +def serve_fake(fake, host, port, after_request=None): + from . import serve + return serve(fake, host, port, after_request=after_request) + + +if __name__ == "__main__": + main() diff --git a/simplyblock_core/storage_arrays/testing/dell_powerflex.py b/simplyblock_core/storage_arrays/testing/dell_powerflex.py new file mode 100644 index 0000000000..b6897db25e --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/dell_powerflex.py @@ -0,0 +1,312 @@ +"""Fake Dell PowerFlex REST gateway (``/api/login`` token auth on 3.6 and +4.x, 4.x ``/rest/auth/login`` OAuth), the subset the adapter uses. + +Quirks reproduced: the ``/api/login`` token is a JSON *string*, sent as +basic auth ``user:`` on 3.x and as ``Bearer `` on 4.x +(goscaleio); ``/api/version`` answers a JSON string (``"3.6"`` / ``"4.5"``) +and accepts the basic-auth token on both; 3.x answers 404 for +``/rest/auth/login``; "not found" is +HTTP 500 with ``{"message", "httpStatusCode", "errorCode"}``; sizes round up +to 8 GiB; ``mappedSdcInfo`` is ``null`` when unmapped; a second SDC mapping +needs ``allowMultipleMappings``; SDCs exist before (and independently of) +any host registration. Error codes are placeholders, not PowerFlex's. +""" +from __future__ import annotations + +import json +import secrets +import uuid +from typing import Any + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +GIB = 1 << 30 +_KB_PER_8GIB = 8 * 1024 * 1024 +_VOL = r"/api/instances/Volume::(?P[0-9a-zA-Z]+)" + + +def _err(msg: str, status: int = 500, code: int = 0) -> FakeError: + return FakeError(status, {"message": msg, "httpStatusCode": status, "errorCode": code}) + + +def _bool(v: Any) -> bool: + return str(v).upper() == "TRUE" + + +class FakePowerFlex(FakeArray): + vendor = "dell-powerflex" + + def __init__(self, username: str = "admin", password: str = "pf-pass", + generation: int = 4): + super().__init__() + if generation not in (3, 4): + raise ValueError("generation must be 3 or 4") + self.generation = generation + self.username = username + self.password = password + self.tokens: set[str] = set() # /api/login session tokens + self.access_tokens: set[str] = set() # 4.x /rest/auth/login tokens + self.system_id = "4a54e5c8a1b2c3d4" + self.pools = {"7d2e1c5b00000001": {"id": "7d2e1c5b00000001", "name": "sb-pool", + "protectionDomainId": "e1f2a3b400000000"}} + self.volumes: dict[str, dict[str, Any]] = {} + self.sdcs: dict[str, dict[str, Any]] = {} + + # -- fake controls ------------------------------------------------------ + + @property + def pool_id(self) -> str: + return next(iter(self.pools)) + + def _new_id(self) -> str: + return f"{secrets.token_hex(4)}{self.next_id():08x}" + + def expire_sessions(self) -> None: + self.tokens.clear() + self.access_tokens.clear() + + def set_credential(self, **fields: str) -> None: + self.username = fields.get("username", self.username) + self.password = fields["password"] + + def add_sdc(self, guid: str, name: str | None = None) -> str: + sid = self._new_id() + self.sdcs[sid] = {"id": sid, "sdcGuid": guid, "name": name, + "sdcIp": f"10.0.0.{len(self.sdcs) + 10}", "sdcApproved": True, + "mdmConnectionState": "Connected", + "links": [{"rel": "self", "href": f"/api/instances/Sdc::{sid}"}]} + return sid + + def add_volume(self, name: str | None, size_bytes: int, pool_id: str | None = None) -> str: + vid = self._new_id() + kb = -(-max(size_bytes // 1024, 1) // _KB_PER_8GIB) * _KB_PER_8GIB + self.volumes[vid] = { + "id": vid, "name": name, "sizeInKb": kb, + "storagePoolId": pool_id or self.pool_id, "volumeType": "ThinProvisioned", + "vtreeId": self._new_id(), "ancestorVolumeId": None, "mappedSdcInfo": None, + "creationTime": 1763600000, "isObfuscated": False, + "links": [{"rel": "self", "href": f"/api/instances/Volume::{vid}"}], + } + return vid + + # -- auth --------------------------------------------------------------- + + def _auth(self, req: Request, basic_ok: bool = False) -> None: + if self.generation == 3 or basic_ok: + creds = req.basic_auth() + if creds and creds[1] in self.tokens: + return + if self.generation >= 4: + header = req.headers.get("Authorization", "") + if header.startswith("Bearer ") and (header[7:] in self.access_tokens + or header[7:] in self.tokens): + return + raise _err("Unauthorized", 401) + + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return [] # PowerFlex presents through its SDC, not iSCSI + + def lio_all_iqns(self) -> list[str]: + return sorted(set()) + + def lio_host_iqns(self, host_id: str) -> list[str]: + return [] + + @route("GET", "/api/version") + def api_version(self, req): + self._auth(req, basic_ok=True) + version = "3.6" if self.generation == 3 else "4.5" + return Reply(200, json.dumps(version).encode(), {"Content-Type": "application/json"}) + + @route("GET", "/api/login") + def api_login(self, req): + if req.basic_auth() != (self.username, self.password): + raise _err("Unauthorized", 401) + token = secrets.token_urlsafe(24) + self.tokens.add(token) + # The gateway returns the token as a JSON string literal. + return Reply(200, json.dumps(token).encode(), {"Content-Type": "application/json"}) + + @route("GET", "/api/logout") + def api_logout(self, req): + creds = req.basic_auth() + if creds: + self.tokens.discard(creds[1]) + header = req.headers.get("Authorization", "") + if header.startswith("Bearer "): + self.tokens.discard(header[7:]) + return Reply(200, None) + + @route("POST", "/rest/auth/login") + def rest_login(self, req): + if self.generation != 4: + return Reply(404, b"HTTP Status 404 - Not Found", + {"Content-Type": "text/html"}) + body = req.json() or {} + if (body.get("username"), body.get("password")) != (self.username, self.password): + raise FakeError(401, {"error": "invalid_grant", + "error_description": "Invalid user credentials"}) + token = secrets.token_urlsafe(32) + self.access_tokens.add(token) + return Reply(200, {"access_token": token, "expires_in": 300, + "refresh_token": secrets.token_urlsafe(32), + "refresh_expires_in": 1800, "token_type": "Bearer", + "scope": "openid profile email"}) + + @route("POST", "/rest/auth/logout") + def rest_logout(self, req): + header = req.headers.get("Authorization", "") + self.access_tokens.discard(header[7:]) + return Reply(204, None) + + # -- system ----------------------------------------------------------------- + + @route("GET", "/api/types/System/instances") + def systems(self, req): + self._auth(req) + version = "R3_6.0.0" if self.generation == 3 else "R4_5.0.0" + return Reply(200, [{"id": self.system_id, "name": "fake-pflex", + "systemVersionName": f"DellEMC PowerFlex Version: {version}", + "mdmClusterState": "ClusteredNormal"}]) + + # -- volumes ------------------------------------------------------------- + + def _vol(self, vid: str) -> dict[str, Any]: + if vid not in self.volumes: + raise _err("Could not find the volume", 500, 79) + return self.volumes[vid] + + @route("POST", "/api/types/Volume/instances") + def create_volume(self, req): + self._auth(req) + body = req.json() or {} + name = body.get("name") + if name and len(name) > 31: + raise _err("Name is too long", 500, 7) + if name and any(v["name"] == name for v in self.volumes.values()): + raise _err("Volume name already in use", 500, 6) + pool = body.get("storagePoolId") + if pool not in self.pools: + raise _err("Could not find the Storage Pool", 500, 61) + if body.get("volumeType") not in ("ThinProvisioned", "ThickProvisioned"): + raise _err("Invalid volume type", 400, 1) + try: + kb = int(body["volumeSizeInKb"]) + except (KeyError, TypeError, ValueError): + raise _err("Invalid volume size", 400, 1) from None + vid = self.add_volume(name, kb * 1024, pool) + return Reply(200, {"id": vid}) + + @route("GET", _VOL) + def get_volume(self, req, vid): + self._auth(req) + return Reply(200, self._vol(vid)) + + @route("POST", "/api/types/Volume/instances/action/queryIdByKey") + def query_id(self, req): + self._auth(req) + name = (req.json() or {}).get("name") + for v in self.volumes.values(): + if v["name"] == name: + return Reply(200, v["id"]) + raise _err("Could not find the volume", 500, 79) + + @route("POST", _VOL + "/action/(?P[A-Za-z]+)") + def volume_action(self, req, vid, action): + self._auth(req) + v = self._vol(vid) + body = req.json() or {} + mapped = v["mappedSdcInfo"] or [] + if action == "setVolumeSize": + gb = int(body["sizeInGB"]) + kb = -(-gb // 8) * 8 * 1024 * 1024 + if kb < v["sizeInKb"]: + raise _err("Volume size cannot be decreased", 500, 76) + v["sizeInKb"] = kb + return Reply(200, {}) + if action == "removeVolume": + if body.get("removeMode") not in ("ONLY_ME", "INCLUDING_DESCENDANTS", + "DESCENDANTS_ONLY", "WHOLE_VTREE"): + raise _err("Invalid remove mode", 400, 1) + if mapped: + raise _err("The volume is mapped to an SDC; unmap it first", 500, 80) + del self.volumes[vid] + return Reply(200, {}) + if action == "addMappedSdc": + sid = body.get("sdcId") + if sid not in self.sdcs: + raise _err("Could not find the SDC", 500, 100) + if any(m["sdcId"] == sid for m in mapped): + raise _err("The volume is already mapped to this SDC", 500, 81) + if mapped and not _bool(body.get("allowMultipleMappings")): + raise _err("The volume is already mapped to another SDC; " + "set allowMultipleMappings", 500, 82) + sdc = self.sdcs[sid] + v["mappedSdcInfo"] = mapped + [{ + "sdcId": sid, "sdcIp": sdc["sdcIp"], "sdcName": sdc["name"], + "limitIops": 0, "limitBwInMbps": 0, + "accessMode": body.get("accessMode", "ReadWrite")}] + return Reply(200, {}) + if action == "removeMappedSdc": + sid = body.get("sdcId") + rest = [m for m in mapped if m["sdcId"] != sid] + if len(rest) == len(mapped): + raise _err("The volume is not mapped to SDC", 500, 83) + v["mappedSdcInfo"] = rest or None + return Reply(200, {}) + raise _err(f"Unknown action {action}", 400, 1) + + # -- SDCs ----------------------------------------------------------------- + + @route("GET", "/api/types/Sdc/instances") + def list_sdcs(self, req): + self._auth(req) + return Reply(200, list(self.sdcs.values())) + + @route("POST", r"/api/instances/Sdc::(?P[0-9a-zA-Z]+)/action/setSdcName") + def set_sdc_name(self, req, sid): + self._auth(req) + if sid not in self.sdcs: + raise _err("Could not find the SDC", 500, 100) + name = (req.json() or {}).get("sdcName") + if any(s["name"] == name and s["id"] != sid for s in self.sdcs.values()): + raise _err("SDC name already in use", 500, 6) + self.sdcs[sid]["name"] = name + return Reply(200, {}) + + +def _guid() -> str: + return str(uuid.uuid4()).upper() + + +def contract_setup(generation: int = 4) -> ContractSetup: + fake = FakePowerFlex(username="admin", password="pf-pass", generation=generation) + guid_a, guid_b = _guid(), _guid() + # SDCs register themselves with the MDM when installed on a host; the + # adapter only finds them, so the fake starts with both hosts' SDCs. + fake.add_sdc(guid_a) + fake.add_sdc(guid_b) + return ContractSetup( + fake=fake, + config=make_config("dell-powerflex", "https://powerflex.fake.example", + storage_pool_id=fake.pool_id), + credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("pf-pass")}), + bad_credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.SDC: ( + (Initiator(protocol=Protocol.SDC, address=guid_a),), + (Initiator(protocol=Protocol.SDC, address=guid_b),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_volume(name, size), + has_lun_numbers=False, + ) diff --git a/simplyblock_core/storage_arrays/testing/dell_powermax.py b/simplyblock_core/storage_arrays/testing/dell_powermax.py new file mode 100644 index 0000000000..2c4903481b --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/dell_powermax.py @@ -0,0 +1,612 @@ +"""Fake Dell PowerMax Unisphere REST API 10.x (the subset the adapter uses). + +Quirks reproduced: basic auth on every request (401 plain-text body); +``{"message": "..."}`` error bodies, 400 for most failures, 404 for missing +objects ("Could not find volume", "... cannot be found"), 409 for "already +exists", 500 for a masking view naming a missing port group; NVMe host +initiators as ``:<32 upper-case hex host ID>``; versioned paths (``/univmax/restapi/{ver}/...``) with +unknown versions refused; sizes in cylinders (1 CYL = 1920 KiB), also +accepting MB/GB/TB; ``executionOption: ASYNCHRONOUS`` answered with +``202`` + job that runs on the second poll of ``/{ver}/system/job/{id}`` +(``SCHEDULED`` -> ``RUNNING`` -> ``SUCCEEDED`` or ``job_failure_status`` +with the error in ``result``); volume listing as an iterator (``count``/``resultList``, no +``resultList`` when empty); non-unique volume identifiers; masked storage +groups cannot lose their last device; host LUN addresses in hex; port groups +bound to one protocol. +""" +from __future__ import annotations + +import math +import re +import secrets +from collections.abc import Callable +from typing import Any + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +SYMMETRIX_ID = "000197900123" +CYL_BYTES = 1920 * 1024 +_ROOT = "/univmax/restapi" +_V = _ROOT + r"/(?P\d+)" +_SLO = _V + r"/sloprovisioning/symmetrix/(?P\d{12})" +_NAME_RE = re.compile(r"^[A-Za-z0-9_-]{1,64}$") +_UNIT_MB = {"MB": 1, "GB": 1024, "TB": 1024 * 1024} + + +def _err(status: int, message: str) -> FakeError: + return FakeError(status, {"message": message}) + + +def _bad(message: str) -> FakeError: + return _err(400, message) + + +def _proto_of(initiator: str) -> str: + v = initiator.lower() + if v.startswith(("iqn.", "eui.")): + return "iscsi" + if v.startswith("nqn."): + return "nvme-tcp" + return "fc" + + +class FakeDellPowerMax(FakeArray): + vendor = "dell-powermax" + api_versions = ("100", "101") + + def __init__(self, username: str = "smc", password: str = "pmax-pass", + symmetrix_id: str = SYMMETRIX_ID): + super().__init__() + self.username = username + self.password = password + self.symmetrix_id = symmetrix_id + self.volumes: dict[str, dict[str, Any]] = {} # device id -> volume + self.storage_groups: dict[str, dict[str, Any]] = {} # name -> sg + self.hosts: dict[str, dict[str, Any]] = {} # name -> host + self.port_groups: dict[str, dict[str, Any]] = { + "sb_pg_iscsi": {"protocol": "iscsi", "ports": ["SE-1E:0", "SE-2E:0"]}, + "sb_pg_fc": {"protocol": "fc", "ports": ["FA-1D:4", "FA-2D:4"]}, + "sb_pg_nvme": {"protocol": "nvme-tcp", "ports": ["OR-1C:2", "OR-2C:2"]}, + } + self.views: dict[str, dict[str, Any]] = {} # name -> masking view + self.jobs: dict[str, dict[str, Any]] = {} + self._job_fns: dict[str, Callable[[], Reply]] = {} + self.first_lun = 1 + self.job_failure_status = "FAILED" + self._dev = 0x00100 + + # -- fake controls ------------------------------------------------------ + + def set_credential(self, **fields: str) -> None: + self.username = fields.get("username", self.username) + self.password = fields.get("password", self.password) + + def _new_device(self, identifier: str, cyl: int) -> str: + self._dev += 1 + dev = f"{self._dev:05X}" + wwn = ("6000097000" + self.symmetrix_id[-12:] + dev.encode().hex()).upper() + self.volumes[dev] = { + "volumeId": dev, "type": "TDEV", "emulation": "FBA", "ssid": "FFFFFFFF", + "status": "Ready", "reserved": False, "pinned": False, + "volume_identifier": identifier, "cap_cyl": cyl, "wwn": wwn, + "effective_wwn": wwn, "has_effective_wwn": False, "encapsulated": False, + "storageGroupId": [], "snapvx_source": False, "snapvx_target": False, + "mobility_id_enabled": False, + } + return dev + + def add_volume(self, name: str, size: int, storage_group: str | None = None) -> str: + """Create a device directly (optionally inside a storage group).""" + dev = self._new_device(name, math.ceil(size / CYL_BYTES)) + if storage_group: + sg = self.storage_groups.setdefault(storage_group, self._sg_record( + storage_group, "SRP_1", None)) + sg["volumes"].append(dev) + self.volumes[dev]["storageGroupId"].append(storage_group) + return dev + + # -- dispatch helpers ---------------------------------------------------------- + + def _auth(self, req: Request) -> None: + if req.basic_auth() != (self.username, self.password): + raise FakeError(401, "Unauthorized", {"WWW-Authenticate": 'Basic realm="Unisphere"'}) + + def _check(self, req: Request, ver: str, sym: str | None = None) -> None: + self._auth(req) + if ver not in self.api_versions: + raise _err(404, f"Unsupported REST API version {ver}") + if sym is not None and sym != self.symmetrix_id: + raise _err(404, f"Cannot find Symmetrix {sym}") + + def _run(self, req: Request, fn: Callable[[], Reply]) -> Reply: + """Honour ``executionOption``: run now, or queue as a job.""" + body = req.json() or {} + option = body.get("executionOption", "SYNCHRONOUS") + if option not in ("SYNCHRONOUS", "ASYNCHRONOUS"): + raise _bad(f"Invalid executionOption {option}") + if option == "SYNCHRONOUS": + return fn() + job_id = str(1_000_000 + self.next_id()) + self.jobs[job_id] = { + "jobId": job_id, "name": f"{req.method} {req.path}", "status": "SCHEDULED", + "username": self.username, "result": None, + "last_modified_date": "Nov-20-2026 00:00:00.000", + "resourceLink": f"https://unisphere.fake.example{req.path}", + "task": [{"execution_order": 1, "description": f"{req.method} {req.path}"}], + "_polls": 0, + } + self._job_fns[job_id] = fn + return Reply(202, self._public_job(job_id)) + + def _public_job(self, job_id: str) -> dict[str, Any]: + return {k: v for k, v in self.jobs[job_id].items() if not k.startswith("_")} + + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return list(self.volumes) + + def lio_all_iqns(self) -> list[str]: + return sorted({str(i).lower() for h in self.hosts.values() for i in h.get("initiator", []) if str(i).lower().startswith("iqn.")}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + host = self.hosts.get(host_id) or {} + return [str(i).lower() for i in host.get("initiator", []) if str(i).lower().startswith("iqn.")] + + @route("GET", _V + r"/system/job/(?P\d+)") + def get_job(self, req, ver, job_id): + self._check(req, ver) + job = self.jobs.get(job_id) + if job is None: + raise _err(404, f"Cannot find job {job_id}") + job["_polls"] += 1 + if job["status"] == "SCHEDULED": + job["status"] = "RUNNING" + elif job["status"] == "RUNNING": + fn = self._job_fns.pop(job_id) + try: + fn() + job["status"], job["result"] = "SUCCEEDED", "Succeeded" + except FakeError as e: + body = e.reply.body if isinstance(e.reply.body, dict) else {} + job["status"] = self.job_failure_status + job["result"] = f"Failed : {body.get('message', e.reply.status)}" + return Reply(200, self._public_job(job_id)) + + # -- system ------------------------------------------------------------------------- + + @route("GET", _ROOT + "/version") + def version(self, req): + self._auth(req) + return Reply(200, {"version": "T10.1.0.0", "api_version": "101", + "supported_api_versions": list(self.api_versions)}) + + @route("GET", _V + r"/system/symmetrix/(?P\d{12})") + def get_symmetrix(self, req, ver, sym): + self._check(req, ver, sym) + return Reply(200, {"symmetrixId": sym, "device_count": len(self.volumes), + "ucode": "6079.225.0", "model": "PowerMax_8500", + "local": True, "default_fba_srp": "SRP_1"}) + + # -- volumes ------------------------------------------------------------------------ + + def _vol(self, dev: str) -> dict[str, Any]: + if dev not in self.volumes: + raise _err(404, f"Could not find volume {dev}") + return self.volumes[dev] + + def _vol_view(self, v: dict[str, Any]) -> dict[str, Any]: + cyl = v["cap_cyl"] + mapped = any(v["volumeId"] in self.storage_groups[sg]["volumes"] + and self.storage_groups[sg]["maskingview"] for sg in v["storageGroupId"]) + return {**v, + "cap_gb": round(cyl * 1920 / 1024 / 1024, 2), + "cap_mb": round(cyl * 1.875, 2), + "num_of_storage_groups": len(v["storageGroupId"]), + "num_of_front_end_paths": 0, "mapped": mapped, + "allocated_percent": 0} + + @staticmethod + def _cylinders(attrs: dict[str, Any]) -> int: + unit = attrs.get("capacityUnit", "GB") + try: + size = float(attrs["volume_size"]) + except (KeyError, TypeError, ValueError) as e: + raise _bad("Invalid volume_size") from e + if size <= 0: + raise _bad("volume_size must be positive") + if unit == "CYL": + if size != int(size): + raise _bad("volume_size in CYL must be a whole number") + return int(size) + if unit not in _UNIT_MB: + raise _bad(f"Invalid capacityUnit {unit}") + return math.ceil(size * _UNIT_MB[unit] / 1.875) + + def _identifier(self, attrs: dict[str, Any]) -> str: + ident = attrs.get("volumeIdentifier") or {} + if not ident: + return "" + if ident.get("volumeIdentifierChoice") != "identifier_name": + raise _bad("Unsupported volumeIdentifierChoice") + name = ident.get("identifier_name") or "" + if len(name) > 64: + raise _bad("The volume identifier must not exceed 64 characters") + return name + + @route("GET", _SLO + "/volume") + def list_volumes(self, req, ver, sym): + self._check(req, ver, sym) + rows = list(self.volumes.values()) + if req.q("volume_identifier") is not None: + want = req.q("volume_identifier") or "" + if want.startswith(""): + rows = [v for v in rows if want[6:] in v["volume_identifier"]] + else: + rows = [v for v in rows if v["volume_identifier"] == want] + if req.q("storageGroupId") is not None: + sg = req.q("storageGroupId") + rows = [v for v in rows if sg in v["storageGroupId"]] + body: dict[str, Any] = {"count": len(rows), "maxPageSize": 1000, + "expirationTime": 1_900_000_000_000} + if rows: + body["id"] = f"{secrets.token_hex(4)}-iterator" + body["resultList"] = {"result": [{"volumeId": v["volumeId"]} for v in rows], + "from": 1, "to": len(rows)} + return Reply(200, body) + + @route("GET", _SLO + r"/volume/(?P[0-9A-Fa-f]+)") + def get_volume(self, req, ver, sym, dev): + self._check(req, ver, sym) + return Reply(200, self._vol_view(self._vol(dev))) + + @route("PUT", _SLO + r"/volume/(?P[0-9A-Fa-f]+)") + def put_volume(self, req, ver, sym, dev): + self._check(req, ver, sym) + vol = self._vol(dev) + body = req.json() or {} + expand = ((body.get("editVolumeActionParam") or {}).get("expandVolumeParam") or {}) + if not expand: + raise _bad("Unsupported editVolumeActionParam") + cyl = self._cylinders(expand.get("volumeAttribute") or {}) + + def do() -> Reply: + if cyl < vol["cap_cyl"]: + raise _bad(f"The new size of volume {dev} must be larger than the " + "current size") + vol["cap_cyl"] = cyl + return Reply(200, self._vol_view(vol)) + return self._run(req, do) + + @route("DELETE", _SLO + r"/volume/(?P[0-9A-Fa-f]+)") + def delete_volume(self, req, ver, sym, dev): + self._check(req, ver, sym) + vol = self._vol(dev) + if vol["storageGroupId"]: + raise _bad(f"A problem occurred deleting the volume {dev}: the device is a " + f"member of storage group {vol['storageGroupId'][0]}") + del self.volumes[dev] + return Reply(204) + + # -- storage groups ------------------------------------------------------------------ + + @staticmethod + def _sg_record(name: str, srp: str, slo: str | None) -> dict[str, Any]: + return {"storageGroupId": name, "srp": srp, "slo": slo or "NONE", + "volumes": [], "maskingview": []} + + def _sg(self, name: str) -> dict[str, Any]: + if name not in self.storage_groups: + raise _err(404, f"The requested storage group {name} cannot be found") + return self.storage_groups[name] + + def _sg_view(self, sg: dict[str, Any]) -> dict[str, Any]: + cyl = sum(self.volumes[d]["cap_cyl"] for d in sg["volumes"]) + out = {"storageGroupId": sg["storageGroupId"], "srp": sg["srp"], "slo": sg["slo"], + "num_of_vols": len(sg["volumes"]), "num_of_child_sgs": 0, + "num_of_parent_sgs": 0, "num_of_masking_views": len(sg["maskingview"]), + "num_of_snapshots": 0, "cap_gb": round(cyl * 1920 / 1024 / 1024, 2), + "device_emulation": "FBA", "type": "Standalone", "unprotected": True, + "compression": True} + if sg["maskingview"]: + out["maskingview"] = list(sg["maskingview"]) + return out + + def _add_devices(self, sg: dict[str, Any], attrs_list: list[dict[str, Any]]) -> None: + for attrs in attrs_list: + cyl = self._cylinders(attrs) + ident = self._identifier(attrs) + for _ in range(int(attrs.get("num_of_vols", 1))): + dev = self._new_device(ident, cyl) + sg["volumes"].append(dev) + self.volumes[dev]["storageGroupId"].append(sg["storageGroupId"]) + + @route("POST", _SLO + "/storagegroup") + def post_sg(self, req, ver, sym): + self._check(req, ver, sym) + body = req.json() or {} + name = body.get("storageGroupId") or "" + if not _NAME_RE.match(name): + raise _bad(f"Invalid storage group name {name!r}") + srp = body.get("srpId", "None") + if srp not in ("SRP_1", "None"): + raise _err(404, f"Cannot find SRP {srp}") + params = body.get("sloBasedStorageGroupParam") or [] + for p in params: + if p.get("sloId", "None") not in ("None", "Diamond", "Platinum", "Gold", + "Silver", "Bronze", "Optimized"): + raise _bad(f"Invalid sloId {p.get('sloId')}") + if "workloadSelection" in p and p["workloadSelection"] not in ("None", None): + raise _bad("workloadSelection is not supported on this array") + + def do() -> Reply: + if name in self.storage_groups: + raise _err(409, "The requested storage group resource already exists") + slo = next((p.get("sloId") for p in params if p.get("sloId") != "None"), None) + sg = self._sg_record(name, srp, slo) + self.storage_groups[name] = sg + try: + for p in params: + self._add_devices(sg, p.get("volumeAttributes") or []) + except FakeError: + del self.storage_groups[name] + raise + return Reply(200, self._sg_view(sg)) + return self._run(req, do) + + @route("GET", _SLO + r"/storagegroup/(?P[^/]+)") + def get_sg(self, req, ver, sym, name): + self._check(req, ver, sym) + return Reply(200, self._sg_view(self._sg(name))) + + @route("PUT", _SLO + r"/storagegroup/(?P[^/]+)") + def put_sg(self, req, ver, sym, name): + self._check(req, ver, sym) + sg = self._sg(name) + edit = (req.json() or {}).get("editStorageGroupActionParam") or {} + if "removeVolumeParam" in edit: + devs = edit["removeVolumeParam"].get("volumeId") or [] + + def remove() -> Reply: + for d in devs: + if d not in sg["volumes"]: + raise _bad(f"Storage Group {name} does not contain volume {d}") + if sg["maskingview"] and len(set(sg["volumes"]) - set(devs)) == 0: + raise _bad(f"Cannot remove the last volume from Storage Group {name} " + "because it is part of a masking view") + for d in devs: + sg["volumes"].remove(d) + self.volumes[d]["storageGroupId"].remove(name) + return Reply(200, self._sg_view(sg)) + return self._run(req, remove) + expand = edit.get("expandStorageGroupParam") or {} + add = expand.get("addVolumeParam") + if add is not None: + if not add.get("create_new_volumes", False): + raise _bad("Only create_new_volumes is supported by the fake") + + def grow() -> Reply: + self._add_devices(sg, add.get("volumeAttributes") or []) + return Reply(200, self._sg_view(sg)) + return self._run(req, grow) + raise _bad("Unsupported editStorageGroupActionParam") + + @route("DELETE", _SLO + r"/storagegroup/(?P[^/]+)") + def delete_sg(self, req, ver, sym, name): + self._check(req, ver, sym) + sg = self._sg(name) + if sg["maskingview"]: + raise _bad(f"Storage Group {name} is part of masking view {sg['maskingview'][0]}") + if sg["volumes"]: + raise _bad(f"Storage Group {name} is not empty") + del self.storage_groups[name] + return Reply(204) + + # -- hosts ------------------------------------------------------------------------------- + + def _host(self, name: str) -> dict[str, Any]: + if name not in self.hosts: + raise _err(404, f"Host {name} cannot be found") + return self.hosts[name] + + def _host_view(self, h: dict[str, Any]) -> dict[str, Any]: + return {"hostId": h["hostId"], "num_of_masking_views": len(h["maskingview"]), + "num_of_initiators": len(h["initiator"]), "num_of_host_groups": 0, + "port_flags_override": False, "consistent_lun": False, + "initiator": list(h["initiator"]), "maskingview": list(h["maskingview"]), + "type": {"fc": "Fibre", "iscsi": "iSCSI", "nvme-tcp": "NVMe/TCP"}.get( + _proto_of(h["initiator"][0]) if h["initiator"] else "", "")} + + def _claim(self, host: str, initiators: list[str]) -> list[str]: + out = [] + for i in initiators: + proto = _proto_of(i) + if proto == "fc" and not re.match(r"^[0-9a-f]{16}$", i): + raise _bad(f"Invalid initiator {i}: expected a 16-digit WWN") + if proto == "nvme-tcp" and not re.match(r"^nqn\..+:[0-9A-F]{32}$", i): + raise _bad(f"Invalid initiator {i}: expected :") + for other in self.hosts.values(): + if i.lower() in (x.lower() for x in other["initiator"]): + if other["hostId"] == host: + raise _bad(f"Initiator {i} is already in Host {host}") + raise _bad(f"Initiator {i} is already in use by Host {other['hostId']}") + out.append(i) + protos = {_proto_of(x) for x in out + (self.hosts[host]["initiator"] + if host in self.hosts else [])} + if len(protos) > 1: + raise _bad("A host cannot mix initiator types") + return out + + @route("POST", _SLO + "/host") + def post_host(self, req, ver, sym): + self._check(req, ver, sym) + body = req.json() or {} + name = body.get("hostId") or "" + if not _NAME_RE.match(name): + raise _bad(f"Invalid host name {name!r}") + initiators = body.get("initiatorId") or [] + + def do() -> Reply: + if name in self.hosts: + raise _err(409, f"Host {name} already exists") + claimed = self._claim(name, initiators) + self.hosts[name] = {"hostId": name, "initiator": claimed, "maskingview": []} + return Reply(200, self._host_view(self.hosts[name])) + return self._run(req, do) + + @route("GET", _SLO + r"/host/(?P[^/]+)") + def get_host(self, req, ver, sym, name): + self._check(req, ver, sym) + return Reply(200, self._host_view(self._host(name))) + + @route("PUT", _SLO + r"/host/(?P[^/]+)") + def put_host(self, req, ver, sym, name): + self._check(req, ver, sym) + host = self._host(name) + edit = (req.json() or {}).get("editHostActionParam") or {} + add = (edit.get("addInitiatorParam") or {}).get("initiator") + if add is None: + raise _bad("Unsupported editHostActionParam") + + def do() -> Reply: + host["initiator"].extend(self._claim(name, add)) + return Reply(200, self._host_view(host)) + return self._run(req, do) + + @route("DELETE", _SLO + r"/host/(?P[^/]+)") + def delete_host(self, req, ver, sym, name): + self._check(req, ver, sym) + host = self._host(name) + if host["maskingview"]: + raise _bad(f"Host {name} is part of masking view {host['maskingview'][0]}") + del self.hosts[name] + return Reply(204) + + # -- masking views --------------------------------------------------------------- + + def _view(self, name: str) -> dict[str, Any]: + if name not in self.views: + raise _err(404, f"Masking View {name} cannot be found") + return self.views[name] + + @route("POST", _SLO + "/maskingview") + def post_view(self, req, ver, sym): + self._check(req, ver, sym) + body = req.json() or {} + name = body.get("maskingViewId") or "" + if not _NAME_RE.match(name): + raise _bad(f"Invalid masking view name {name!r}") + host = ((body.get("hostOrHostGroupSelection") or {}) + .get("useExistingHostParam") or {}).get("hostId") + pg = ((body.get("portGroupSelection") or {}) + .get("useExistingPortGroupParam") or {}).get("portGroupId") + sg = ((body.get("storageGroupSelection") or {}) + .get("useExistingStorageGroupParam") or {}).get("storageGroupId") + if not (host and pg and sg): + raise _bad("A masking view needs an existing host, port group and storage group") + + def do() -> Reply: + if name in self.views: + raise _err(409, "The requested masking view resource already exists") + h, s = self._host(host), self._sg(sg) + if pg not in self.port_groups: + raise _err(500, "Port Group on Symmetrix cannot be found") + if not s["volumes"]: + raise _bad(f"Storage Group {sg} is empty") + protos = {_proto_of(i) for i in h["initiator"]} + if protos and protos != {self.port_groups[pg]["protocol"]}: + raise _bad(f"Port Group {pg} does not match the protocol of Host {host}") + used = {lun for v in self.views.values() if v["host"] == host + for lun in v["luns"].values()} + luns = {} + for dev in s["volumes"]: + lun = next(n for n in range(self.first_lun, 4096) if n not in used) + used.add(lun) + luns[dev] = lun + self.views[name] = {"maskingViewId": name, "host": host, "pg": pg, "sg": sg, + "luns": luns} + h["maskingview"].append(name) + s["maskingview"].append(name) + return Reply(200, self._view_view(self.views[name])) + return self._run(req, do) + + @staticmethod + def _view_view(v: dict[str, Any]) -> dict[str, Any]: + return {"maskingViewId": v["maskingViewId"], "hostId": v["host"], + "portGroupId": v["pg"], "storageGroupId": v["sg"]} + + @route("GET", _SLO + r"/maskingview/(?P[^/]+)") + def get_view(self, req, ver, sym, name): + self._check(req, ver, sym) + return Reply(200, self._view_view(self._view(name))) + + @route("GET", _SLO + r"/maskingview/(?P[^/]+)/connections") + def get_connections(self, req, ver, sym, name): + self._check(req, ver, sym) + v = self._view(name) + want = req.q("volume_id") + rows = [] + for dev in self.storage_groups[v["sg"]]["volumes"]: + if want and dev != want: + continue + lun = v["luns"].get(dev) + if lun is None: # device added after the view was created + continue + for initiator in self.hosts[v["host"]]["initiator"]: + for port in self.port_groups[v["pg"]]["ports"]: + rows.append({"volumeId": dev, "host_lun_address": f"{lun:04X}", + "cap_gb": round(self.volumes[dev]["cap_cyl"] * 1920 + / 1024 / 1024, 2), + "initiatorId": initiator, "alias": initiator, + "dir_port": port, "logged_in": False, "on_fabric": False}) + return Reply(200, {"maskingViewConnection": rows}) + + @route("DELETE", _SLO + r"/maskingview/(?P[^/]+)") + def delete_view(self, req, ver, sym, name): + self._check(req, ver, sym) + v = self._view(name) + self.hosts[v["host"]]["maskingview"].remove(name) + self.storage_groups[v["sg"]]["maskingview"].remove(name) + del self.views[name] + return Reply(204) + + +def contract_setup(**options: Any) -> ContractSetup: + fake = FakeDellPowerMax(username="smc", password="pmax-pass") + opts: dict[str, Any] = { + "symmetrix_id": SYMMETRIX_ID, + "port_group": {"iscsi": "sb_pg_iscsi", "fc": "sb_pg_fc", "nvme-tcp": "sb_pg_nvme"}, + } + opts.update(options) + return ContractSetup( + fake=fake, + config=make_config("dell-powermax", "https://unisphere.fake.example:8443", **opts), + credential=Credential({"username": SecretStr("smc"), + "password": SecretStr("pmax-pass")}), + bad_credential=Credential({"username": SecretStr("smc"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + Protocol.NVME_TCP: ( + (Initiator(protocol=Protocol.NVME_TCP, + address="nqn.2014-08.org.nvmexpress:uuid:" + "8c8e1a3c-0b7e-4b8e-9d2a-0000000000a1"),), + (Initiator(protocol=Protocol.NVME_TCP, + address="nqn.2014-08.org.nvmexpress:uuid:" + "8c8e1a3c-0b7e-4b8e-9d2a-0000000000b2"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_volume(name, size, "customer_sg"), + ) diff --git a/simplyblock_core/storage_arrays/testing/dell_powerstore.py b/simplyblock_core/storage_arrays/testing/dell_powerstore.py new file mode 100644 index 0000000000..4993abe1b2 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/dell_powerstore.py @@ -0,0 +1,487 @@ +"""Fake Dell PowerStore REST API (the subset the adapter uses). + +Quirks reproduced: basic auth only on ``login_session``; session cookie +``auth_cookie`` plus ``DELL-EMC-TOKEN`` CSRF header required on non-GET +requests (403 without it); PostgREST-style ``select``/``eq.`` filters with +400 on unknown fields and embedded selects (``host(host_initiators)``); +``limit``/``offset`` paging answered with 206 and ``Content-Range``; 201 +``{"id"}`` on create, 204 on modify/delete/attach; 422 with +``{"messages": [...]}`` for semantic errors (a duplicate volume name answers +``name_conflict_status``, 422 or 500 on real arrays); 400 "Host is not +attached to volume" for a detach of an unattached volume; sizes in bytes, a +multiple of 8192; ``wwn``/``nguid`` prefixed ``naa.``/``nguid.``; a volume +gets its NVMe ``nsid`` on its first attach to an NVMe host. +""" +from __future__ import annotations + +import re +import secrets +import uuid +from http.cookies import CookieError, SimpleCookie +from typing import Any + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +API = "/api/rest" +_ID = r"(?P[^/]+)" +_FC_RE = re.compile(r"^([0-9a-f]{2}:){7}[0-9a-f]{2}$", re.IGNORECASE) +MIN_SIZE = 1 << 20 +MAX_SIZE = 256 << 40 +GRANULARITY = 8192 + +_FIELDS = { + "volume": {"id", "name", "description", "type", "wwn", "nguid", "nsid", "size", "state", + "appliance_id", "creation_timestamp", "is_replication_destination", + "performance_policy_id", "protection_policy_id"}, + "host": {"id", "name", "description", "os_type", "host_group_id", "host_initiators", + "host_connectivity"}, + "host_volume_mapping": {"id", "host_id", "host_group_id", "volume_id", + "logical_unit_number"}, + "cluster": {"id", "global_id", "name", "management_address", "state", + "appliance_count"}, + "appliance": {"id", "name", "model", "service_tag", "express_service_code"}, + "software_installed": {"id", "release_version", "build_version", "is_cluster", + "release_timestamp"}, +} + + +# (kind, embedded relation) -> foreign key on the row +_EMBEDS = {("host_volume_mapping", "host"): "host_id", + ("host_volume_mapping", "volume"): "volume_id"} + + +def _err(status: int, message: str, code: str = "0xE04040010005") -> FakeError: + return FakeError(status, {"messages": [{"code": code, "severity": "Error", + "message_l10n": message, "arguments": []}]}) + + +def _not_found(kind: str, oid: str) -> FakeError: + return _err(404, f"The {kind} {oid} was not found.", "0xE04040010005") + + +def _unprocessable(message: str) -> FakeError: + return _err(422, message, "0xE0A07001000C") + + +class FakeDellPowerStore(FakeArray): + vendor = "dell-powerstore" + + def __init__(self, username: str = "admin", password: str = "ps-pass"): + super().__init__() + self.username = username + self.password = password + self.sessions: dict[str, str] = {} # auth_cookie -> CSRF token + self.volumes: dict[str, dict[str, Any]] = {} # id -> volume + self.hosts: dict[str, dict[str, Any]] = {} # id -> host + self.mappings: list[dict[str, Any]] = [] + self.page_limit = 2000 # array-side default/max page size + self.first_lun = 1 + self.next_nsid = 1 + self.name_conflict_status = 422 + + # -- fake controls ------------------------------------------------------ + + def expire_sessions(self) -> None: + self.sessions.clear() + + def set_credential(self, **fields: str) -> None: + self.username = fields.get("username", self.username) + self.password = fields.get("password", self.password) + + def add_volume(self, name: str, size: int) -> str: + vid = str(uuid.uuid4()) + self.volumes[vid] = { + "id": vid, "name": name, "description": None, "type": "Primary", + "wwn": "naa.68ccf098" + secrets.token_hex(12), + "nguid": "nguid." + secrets.token_hex(16), "nsid": None, + "size": size, "state": "Ready", "appliance_id": "A1", + "creation_timestamp": "2026-11-20T00:00:00+00:00", + "is_replication_destination": False, "performance_policy_id": "default_medium", + "protection_policy_id": None, + } + return vid + + # -- auth --------------------------------------------------------------- + + def _session(self, req: Request) -> None: + cookie: str | None = None + try: + jar = SimpleCookie() + jar.load(req.headers.get("Cookie", "")) + if "auth_cookie" in jar: + cookie = jar["auth_cookie"].value + except CookieError: + pass + if cookie is None or cookie not in self.sessions: + raise _err(401, "Unauthorized: the session is invalid or expired.", + "0xE09040010001") + if req.method != "GET" and req.headers.get("DELL-EMC-TOKEN") != self.sessions[cookie]: + raise _err(403, "Forbidden: missing or invalid DELL-EMC-TOKEN.", "0xE09040010003") + + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return list(self.volumes) + + def lio_all_iqns(self) -> list[str]: + return sorted({p["port_name"].lower() for h in self.hosts.values() for p in h["host_initiators"] if p["port_type"] == "iSCSI"}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + for h in self.hosts.values(): + if host_id in (h["id"], h["name"]): + return [p["port_name"].lower() for p in h["host_initiators"] if p["port_type"] == "iSCSI"] + return [] + + @route("GET", API + "/login_session") + def login_session(self, req): + if req.basic_auth() != (self.username, self.password): + raise _err(401, "Unauthorized: invalid credentials.", "0xE09040010001") + cookie, token = secrets.token_hex(16), secrets.token_hex(16) + self.sessions[cookie] = token + return Reply(200, [{"id": secrets.token_hex(8), "user": self.username, + "role_ids": ["1"], "idle_timeout": 3600, + "is_password_change_required": False, + "is_built_in_user": True}], + {"DELL-EMC-TOKEN": token, + "Set-Cookie": f"auth_cookie={cookie}; Path=/; Secure; HttpOnly"}) + + @route("POST", API + "/logout") + def logout(self, req): + self._session(req) + jar = SimpleCookie() + jar.load(req.headers.get("Cookie", "")) + self.sessions.pop(jar["auth_cookie"].value, None) + return Reply(204) + + # -- query engine ------------------------------------------------------------ + + def _query(self, req: Request, kind: str, rows: list[dict[str, Any]]) -> Reply: + fields = _FIELDS[kind] + for key, values in req.query.items(): + if key in ("select", "limit", "offset", "order"): + continue + if key not in fields: + raise _err(400, f"Invalid filter field: {key}.", "0xE04040020009") + m = re.match(r"^eq\.(.*)$", values[0], re.DOTALL) + if not m: + raise _err(400, f"Unsupported filter operator in {key}={values[0]}.", + "0xE04040020009") + want = m.group(1) + rows = [r for r in rows if _as_filter(r.get(key)) == want] + select = req.q("select") + names = self._select(kind, select) if select else ["id"] + offset = int(req.q("offset", "0") or 0) + limit = min(int(req.q("limit", str(self.page_limit)) or self.page_limit), + self.page_limit) + total = len(rows) + page = rows[offset:offset + limit] + body = [self._row(kind, r, names) for r in page] + if offset + len(page) < total: + end = offset + len(page) - 1 + return Reply(206, body, {"Content-Range": f"{offset}-{end}/{total}"}) + return Reply(200, body) + + def _project(self, req: Request, kind: str, row: dict[str, Any]) -> Reply: + select = req.q("select") + names = self._select(kind, select) if select else sorted(_FIELDS[kind]) + return Reply(200, self._row(kind, row, names)) + + @staticmethod + def _select(kind: str, select: str) -> list[str]: + """Top-level select items; ``rel(a,b)`` embeds a related object.""" + names, depth, cur = [], 0, "" + for ch in select: + if ch == "," and depth == 0: + names.append(cur) + cur = "" + continue + depth += (ch == "(") - (ch == ")") + cur += ch + names.append(cur) + bad = [] + for n in names: + m = re.match(r"^(\w+)\(([\w,]+)\)$", n) + if m: + rel, sub = m.group(1), m.group(2).split(",") + if (kind, rel) not in _EMBEDS or any(f not in _FIELDS[rel] for f in sub): + bad.append(n) + elif n not in _FIELDS[kind]: + bad.append(n) + if bad: + raise _err(400, f"Invalid select field: {','.join(bad)}.", "0xE04040020009") + return names + + def _row(self, kind: str, row: dict[str, Any], names: list[str]) -> dict[str, Any]: + out: dict[str, Any] = {} + for n in names: + m = re.match(r"^(\w+)\(([\w,]+)\)$", n) + if not m: + out[n] = row.get(n) + continue + rel, sub = m.group(1), m.group(2).split(",") + store = self.hosts if rel == "host" else self.volumes + target = store.get(row.get(_EMBEDS[(kind, rel)]) or "") + out[rel] = {f: target.get(f) for f in sub} if target else None + return out + + # -- system ------------------------------------------------------------------ + + @route("GET", API + "/cluster") + def get_cluster(self, req): + self._session(req) + return self._query(req, "cluster", [{ + "id": "0", "global_id": "PS4a7b5c3e9f01", "name": "fake-powerstore", + "management_address": "10.0.0.10", "state": "Configured", "appliance_count": 1}]) + + @route("GET", API + "/appliance") + def get_appliance(self, req): + self._session(req) + return self._query(req, "appliance", [{ + "id": "A1", "name": "fake-powerstore-appliance-1", "model": "PowerStore 1200T", + "service_tag": "FAKE123", "express_service_code": "0"}]) + + @route("GET", API + "/software_installed") + def get_software(self, req): + self._session(req) + return self._query(req, "software_installed", [ + {"id": "c1", "release_version": "4.0.0.0", "build_version": "4.0.0.0.5.123", + "is_cluster": True, "release_timestamp": "2026-01-01T00:00:00Z"}, + {"id": "a1", "release_version": "4.0.0.0", "build_version": "4.0.0.0.5.123", + "is_cluster": False, "release_timestamp": "2026-01-01T00:00:00Z"}, + ]) + + # -- volumes ----------------------------------------------------------------- + + def _vol(self, oid: str) -> dict[str, Any]: + if oid not in self.volumes: + raise _not_found("volume", oid) + return self.volumes[oid] + + @staticmethod + def _check_size(size: Any) -> int: + if not isinstance(size, int) or isinstance(size, bool): + raise _err(400, "Invalid value for size.", "0xE04040020009") + if size < MIN_SIZE or size > MAX_SIZE: + raise _err(400, f"Size {size} is out of range ({MIN_SIZE}..{MAX_SIZE}).", + "0xE04040020009") + if size % GRANULARITY: + raise _err(400, f"Size {size} must be a multiple of {GRANULARITY}.", + "0xE04040020009") + return size + + @route("GET", API + "/volume") + def list_volumes(self, req): + self._session(req) + return self._query(req, "volume", list(self.volumes.values())) + + @route("GET", API + "/volume/" + _ID) + def get_volume(self, req, oid): + self._session(req) + return self._project(req, "volume", self._vol(oid)) + + @route("POST", API + "/volume") + def post_volume(self, req): + self._session(req) + body = req.json() or {} + name = body.get("name") + if not name or len(name) > 128: + raise _err(400, "Invalid value for name.", "0xE04040020009") + size = self._check_size(body.get("size")) + if any(v["name"] == name for v in self.volumes.values()): + raise _err(self.name_conflict_status, + f"The name {name} is already in use by another volume.", + "0xE0A07001000C") + vid = self.add_volume(name, size) + if body.get("appliance_id"): + self.volumes[vid]["appliance_id"] = body["appliance_id"] + return Reply(201, {"id": vid}) + + @route("PATCH", API + "/volume/" + _ID) + def patch_volume(self, req, oid): + self._session(req) + vol = self._vol(oid) + body = req.json() or {} + if "size" in body: + size = self._check_size(body["size"]) + if size < vol["size"]: + raise _unprocessable("The new size must be greater than the current size " + "of the volume.") + vol["size"] = size + return Reply(204) + + @route("DELETE", API + "/volume/" + _ID) + def delete_volume(self, req, oid): + self._session(req) + self._vol(oid) + if any(m["volume_id"] == oid for m in self.mappings): + raise _unprocessable(f"Volume {oid} cannot be deleted because it is attached " + "to one or more hosts.") + del self.volumes[oid] + return Reply(204) + + @route("POST", API + "/volume/" + _ID + "/attach") + def attach(self, req, oid): + self._session(req) + self._vol(oid) + body = req.json() or {} + host_id = body.get("host_id") + if not host_id or host_id not in self.hosts: + raise _not_found("host", str(host_id)) + if any(m["volume_id"] == oid and m["host_id"] == host_id for m in self.mappings): + raise _unprocessable(f"Volume {oid} is already attached to host {host_id}.") + used = {m["logical_unit_number"] for m in self.mappings if m["host_id"] == host_id} + lun = body.get("logical_unit_number") + if lun is None: + lun = next(n for n in range(self.first_lun, 16384) if n not in used) + elif lun in used: + raise _unprocessable(f"Logical unit number {lun} is already in use on the host.") + vol = self.volumes[oid] + nvme = all(p["port_type"] == "NVMe" for p in self.hosts[host_id]["host_initiators"]) + if nvme and self.hosts[host_id]["host_initiators"] and vol["nsid"] is None: + vol["nsid"] = self.next_nsid + self.next_nsid += 1 + self.mappings.append({"id": str(uuid.uuid4()), "host_id": host_id, + "host_group_id": None, "volume_id": oid, + "logical_unit_number": lun}) + return Reply(204) + + @route("POST", API + "/volume/" + _ID + "/detach") + def detach(self, req, oid): + self._session(req) + self._vol(oid) + host_id = (req.json() or {}).get("host_id") + before = len(self.mappings) + self.mappings = [m for m in self.mappings + if not (m["volume_id"] == oid and m["host_id"] == host_id)] + if len(self.mappings) == before: + raise _err(400, "Host is not attached to volume.", "0xE0A08001000F") + return Reply(204) + + @route("GET", API + "/host_volume_mapping") + def list_mappings(self, req): + self._session(req) + return self._query(req, "host_volume_mapping", list(self.mappings)) + + # -- hosts ---------------------------------------------------------------------- + + def _host(self, oid: str) -> dict[str, Any]: + if oid not in self.hosts: + raise _not_found("host", oid) + return self.hosts[oid] + + def _claim(self, host_id: str, initiators: Any) -> list[dict[str, Any]]: + if not isinstance(initiators, list): + raise _err(400, "Invalid value for initiators.", "0xE04040020009") + out = [] + for i in initiators: + port_name, port_type = i.get("port_name"), i.get("port_type") + if port_type not in ("iSCSI", "FC", "NVMe"): + raise _err(400, f"Invalid port_type {port_type}.", "0xE04040020009") + if not port_name: + raise _err(400, "Invalid value for port_name.", "0xE04040020009") + if port_type == "FC" and not _FC_RE.match(port_name): + raise _err(400, f"Invalid FC port_name {port_name}.", "0xE04040020009") + for other in self.hosts.values(): + for p in other["host_initiators"]: + if p["port_name"].lower() == port_name.lower(): + if other["id"] == host_id: + raise _unprocessable( + f"The initiator {port_name} is already added to this host.") + raise _unprocessable( + f"The initiator {port_name} is already registered to host " + f"{other['name']}.") + out.append({"port_name": port_name, "port_type": port_type, + "active_sessions": [], "chap_single_username": None, + "chap_mutual_username": None}) + return out + + @route("GET", API + "/host") + def list_hosts(self, req): + self._session(req) + return self._query(req, "host", list(self.hosts.values())) + + @route("GET", API + "/host/" + _ID) + def get_host(self, req, oid): + self._session(req) + return self._project(req, "host", self._host(oid)) + + @route("POST", API + "/host") + def post_host(self, req): + self._session(req) + body = req.json() or {} + name = body.get("name") + if not name or len(name) > 128: + raise _err(400, "Invalid value for name.", "0xE04040020009") + if body.get("os_type") not in ("Windows", "Linux", "ESXi", "AIX", "HP-UX", "Solaris"): + raise _err(400, f"Invalid os_type {body.get('os_type')}.", "0xE04040020009") + if any(h["name"] == name for h in self.hosts.values()): + raise _unprocessable(f"The name {name} is already in use by another host.") + hid = str(uuid.uuid4()) + initiators = self._claim(hid, body.get("initiators") or []) + self.hosts[hid] = {"id": hid, "name": name, "description": None, + "os_type": body["os_type"], "host_group_id": None, + "host_initiators": initiators, + "host_connectivity": "Local_Only"} + return Reply(201, {"id": hid}) + + @route("PATCH", API + "/host/" + _ID) + def patch_host(self, req, oid): + self._session(req) + host = self._host(oid) + body = req.json() or {} + if "initiators" in body: + raise _err(400, "Use add_initiators/remove_initiators to modify initiators.", + "0xE04040020009") + added = self._claim(oid, body.get("add_initiators") or []) + host["host_initiators"].extend(added) + return Reply(204) + + @route("DELETE", API + "/host/" + _ID) + def delete_host(self, req, oid): + self._session(req) + self._host(oid) + if any(m["host_id"] == oid for m in self.mappings): + raise _unprocessable(f"Host {oid} cannot be deleted because it has volumes " + "attached.") + del self.hosts[oid] + return Reply(204) + + +def _as_filter(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + return "" if value is None else str(value) + + +def contract_setup() -> ContractSetup: + fake = FakeDellPowerStore(username="admin", password="ps-pass") + return ContractSetup( + fake=fake, + config=make_config("dell-powerstore", "https://powerstore.fake.example"), + credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("ps-pass")}), + bad_credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:FF:4C:1A:2C"),), + ), + Protocol.NVME_TCP: ( + (Initiator(protocol=Protocol.NVME_TCP, + address="nqn.2014-08.org.nvmexpress:uuid:host-a"),), + (Initiator(protocol=Protocol.NVME_TCP, + address="nqn.2014-08.org.nvmexpress:uuid:host-b"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_volume(name, size), + ) diff --git a/simplyblock_core/storage_arrays/testing/hitachi_vsp.py b/simplyblock_core/storage_arrays/testing/hitachi_vsp.py new file mode 100644 index 0000000000..ce78042cb4 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/hitachi_vsp.py @@ -0,0 +1,656 @@ +"""Fake Hitachi VSP Configuration Manager REST API (the subset the adapter uses). + +Reproduces: both path layouts (embedded ``/ConfigurationManager/v1/objects`` +and server ``.../objects/storages/{storageDeviceId}``), basic-auth sessions +with ``Authorization: Session ``, ``202`` + job objects for every +change with the change applied only when the job runs (after ``job_polls`` +"Running" polls), job failures carrying the Hitachi error body, LDEVs that +read back as ``NOT DEFINED`` after deletion, per-port host groups with +default group 0, FC WWN uniqueness per port (iSCSI names may repeat across +targets, as on the array), LU paths listed on the LDEV, the resource-group +lock (under ``/ConfigurationManager/v1[/{storageDeviceId}]/services``, not +``/objects``), and ``{"errorSource", "message", "solution", "messageId"}`` +errors. ``GET /ldevs`` treats ``count`` as a window of LDEV numbers from +``headLdevId`` (a window past the last LDEV number answers KART40044-E), which +is how Hitachi's Cinder driver pages through it. + +Message ids and SSB codes the adapter acts on follow Hitachi's Cinder driver +(``hbsd_rest_api.py``): KART30013-E (object does not exist), KART40050-E.. +KART40052-E (resource group locked), SSB 2E22/0001 (LDEV already defined), +2E11/2209 (no free LDEV number), B958/015A (LU path already defined), +B958/0947 (another LDEV on that LUN), B957/4184 (too many WWNs). Other SSBs +here are illustrative. +""" +from __future__ import annotations + +import re +import secrets +from collections.abc import Callable +from typing import Any + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +_ROOT = "/ConfigurationManager/v1/objects" +_P = _ROOT + r"(?:/storages/(?P\d{12}))?" +_SVC = r"/ConfigurationManager/v1(?:/(?P\d{12}))?/services" +_MAX_LDEV = 65279 +NOT_DEFINED = "NOT DEFINED" +_UNITS = {"K": 1 << 10, "M": 1 << 20, "G": 1 << 30, "T": 1 << 40} + + +def _body(msg: str, msg_id: str, source: str = "", solution: str = "") -> dict[str, Any]: + return {"errorSource": source, "message": msg, "solution": solution or + "Check the request and the storage system configuration.", + "messageId": msg_id} + + +def _err(status: int, msg: str, msg_id: str = "KART40040-E", source: str = "") -> FakeError: + return FakeError(status, _body(msg, msg_id, source)) + + +def _storage_err(msg: str, ssb1: str = "2E11", ssb2: str = "0001") -> FakeError: + """A failure inside the storage system: in a job's ``error``.""" + body = _body(msg, "KART30000-E") + body["errorCode"] = {"SSB1": ssb1, "SSB2": ssb2} + return FakeError(400, body) + + +class FakeHitachiVsp(FakeArray): + vendor = "hitachi-vsp" + + def __init__(self, username: str = "sbadmin", password: str = "vsp-secret", + serial: int = 451234, model: str = "VSP E990", + ports: dict[str, str] | None = None, pools: tuple[int, ...] = (0,)): + super().__init__() + self.username, self.password = username, password + self.serial = serial + self.model = model + self.storage_device_id = f"{938000000000 + serial}" + self.sessions: dict[str, int] = {} # token -> sessionId + self.pools = set(pools) + self.ldevs: dict[int, dict[str, Any]] = {} # defined LDEVs only + self.ports = ports or {"CL1-A": "FIBRE", "CL2-A": "FIBRE", + "CL1-B": "ISCSI", "CL2-B": "ISCSI"} + #: (port, number) -> {"name", "mode", "wwns": [], "iscsis": [], "luns": {lun: ldev}} + self.groups: dict[tuple[str, int], dict[str, Any]] = {} + for port in self.ports: + self.groups[(port, 0)] = self._group(f"{port[2:].replace('-', '')}-G00", + "LINUX/IRIX") + self.jobs: dict[int, dict[str, Any]] = {} + self._pending: dict[int, Callable[[], list[str]]] = {} + self._polls_left: dict[int, int] = {} + #: "Running" answers before a job executes. + self.job_polls = 1 + #: Model requires the resource-group lock for changes. + self.require_lock = False + self.lock_owner: str | None = None + self.max_count = 16384 + #: Highest LDEV number this model has (GET /ldevs windows beyond it fail). + self.last_ldev = _MAX_LDEV + + @staticmethod + def _group(name: str, mode: str) -> dict[str, Any]: + return {"name": name, "mode": mode, "modeOptions": [], "wwns": [], "iscsis": [], + "luns": {}} + + # -- fake controls ------------------------------------------------------ + + def expire_sessions(self) -> None: + self.sessions.clear() + self.lock_owner = None + + def set_credential(self, **fields: str) -> None: + self.username = fields.get("username", self.username) + self.password = fields["password"] + + def add_ldev(self, label: str, size: int, ldev_id: int | None = None) -> str: + lid = ldev_id if ldev_id is not None else self._free_ldev(0, _MAX_LDEV) + self.ldevs[lid] = {"pool": min(self.pools), "blocks": size // 512, "label": label} + return str(lid) + + def _free_ldev(self, start: int, end: int) -> int: + lid = next((i for i in range(start, end + 1) if i not in self.ldevs), None) + if lid is None: + raise _storage_err("No free LDEV number in the specified range.", "2E11", "2209") + return lid + + def naa(self, lid: int) -> str: + s = f"{self.serial:06X}" + return f"60060E80{s}0030{s}0000{lid:04X}" + + # -- auth / prefix ------------------------------------------------------- + + def _prefix(self, sdid: str | None) -> str: + if sdid is None: + return _ROOT + if sdid != self.storage_device_id: + raise _err(404, f"The storage system {sdid} is not registered.", "KART40053-E") + return f"{_ROOT}/storages/{sdid}" + + def _token(self, req: Request) -> str: + header = req.headers.get("Authorization", "") + token = header[8:] if header.startswith("Session ") else "" + if token not in self.sessions: + raise _err(401, "The session is invalid or has timed out.", "KART40047-E", + req.path) + return token + + def _auth(self, req: Request, sdid: str | None) -> str: + self._prefix(sdid) + return self._token(req) + + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return [str(i) for i in self.ldevs] + + def lio_all_iqns(self) -> list[str]: + return sorted({str(i).lower() for g in self.groups.values() for i in g.get("iscsis", [])}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + out: list[str] = [] + for g in self.groups.values(): + if g["name"] == host_id: + out += [str(i).lower() for i in g.get("iscsis", []) if str(i).lower() not in out] + return out + + @route("POST", _P + "/sessions") + def post_session(self, req, sdid): + self._prefix(sdid) + if req.basic_auth() != (self.username, self.password): + raise _err(401, "Authentication failed.", "KART40046-E", req.path) + token = secrets.token_hex(16) + self.sessions[token] = self.next_id() + return Reply(200, {"token": token, "sessionId": self.sessions[token]}) + + @route("DELETE", _P + r"/sessions/(?P\d+)") + def delete_session(self, req, sdid, sid): + token = self._auth(req, sdid) + if self.sessions.get(token) != int(sid): + raise _err(404, f"Session {sid} does not exist.", "KART30013-E") + del self.sessions[token] + if self.lock_owner == token: + self.lock_owner = None + return Reply(200, {"sessionId": int(sid)}) + + @route("GET", _ROOT + r"/storages/(?Pinstance|\d{12})") + def get_storage(self, req, which): + self._auth(req, None if which == "instance" else which) + return Reply(200, {"storageDeviceId": self.storage_device_id, "model": self.model, + "serialNumber": self.serial, "svpIp": "192.0.2.10", + "dkcMicroVersion": "93-07-23-80/00", "communicationModes": [ + {"communicationMode": "lanConnectionMode"}]}) + + # -- jobs ----------------------------------------------------------------- + + def _job(self, req: Request, sdid: str | None, token: str, + execute: Callable[[], list[str]], *, lock_request: bool = False) -> Reply: + if not lock_request: + if self.lock_owner and self.lock_owner != token: + raise _err(409, "The resource group is locked by another user.", + "KART40052-E") + if self.require_lock and self.lock_owner != token: + raise _storage_err("The resource group is not locked.", "2E10", "A001") + jid = self.next_id() + prefix = self._prefix(sdid) + self.jobs[jid] = { + "jobId": jid, "self": f"{prefix}/jobs/{jid}", "userId": self.username, + "status": "Initializing", "state": "Queued", + "createdTime": "2026-11-20T10:00:00Z", "updatedTime": "2026-11-20T10:00:00Z", + "request": {"requestUrl": f"https://vsp.fake{req.path}", + "requestMethod": req.method, + "requestBody": req.body.decode() if req.body else ""}, + "affectedResources": [], + } + self._pending[jid] = execute + self._polls_left[jid] = self.job_polls + return Reply(202, dict(self.jobs[jid])) + + @route("GET", _P + r"/jobs/(?P\d+)") + def get_job(self, req, sdid, jid): + self._auth(req, sdid) + job = self.jobs.get(int(jid)) + if job is None: + raise _err(404, f"Job {jid} does not exist.", "KART30013-E") + jid_i = int(jid) + if jid_i in self._pending: + if self._polls_left[jid_i] > 0: + self._polls_left[jid_i] -= 1 + job.update(status="Running", state="Started") + else: + execute = self._pending.pop(jid_i) + try: + job.update(status="Completed", state="Succeeded", + affectedResources=execute(), + completedTime="2026-11-20T10:00:05Z") + except FakeError as e: + job.update(status="Completed", state="Failed", + error=e.reply.body, completedTime="2026-11-20T10:00:05Z") + return Reply(200, dict(job)) + + # -- resource-group lock ------------------------------------------------------- + + @route("POST", _SVC + r"/resource-group-service/actions/(?Plock|unlock)/invoke") + def rg_lock(self, req, sdid, act): + token = self._auth(req, sdid) + prefix = (f"/ConfigurationManager/v1/{sdid}/services" if sdid + else "/ConfigurationManager/v1/services") + + def run() -> list[str]: + if act == "lock": + if self.lock_owner not in (None, token): + raise _storage_err("The resource group is locked by another user.", + "2E10", "A002") + self.lock_owner = token + elif self.lock_owner == token: + self.lock_owner = None + return [f"{prefix}/resource-group-service"] + + return self._job(req, sdid, token, run, lock_request=True) + + @route("PUT", _SVC + r"/resource-group-service/actions/(?Plock|unlock)/invoke") + def rg_lock_put(self, req, sdid, act): + # Hitachi's Cinder driver sends the lock as PUT, Ansible as POST. + return self.rg_lock(req, sdid, act) + + # -- LDEVs ------------------------------------------------------------------ + + def _ldev_view(self, lid: int) -> dict[str, Any]: + ldev = self.ldevs.get(lid) + if ldev is None: + return {"ldevId": lid, "emulationType": NOT_DEFINED, "ssid": "0004", + "resourceGroupId": 0} + paths = [{"portId": port, "hostGroupNumber": num, "hostGroupName": g["name"], + "lun": lun} + for (port, num), g in sorted(self.groups.items()) + for lun, target in sorted(g["luns"].items()) if target == lid] + view = { + "ldevId": lid, "clprId": 0, "emulationType": "OPEN-V-CVS", + "byteFormatCapacity": f"{ldev['blocks'] * 512 / (1 << 30):.2f} G", + "blockCapacity": ldev["blocks"], "numOfPorts": len(paths), + "attributes": ["CVS", "HDP"], "label": ldev["label"], "status": "NML", + "mpBladeId": 0, "ssid": "0004", "poolId": ldev["pool"], + "numOfUsedBlock": 0, "isFullAllocationEnabled": False, + "resourceGroupId": 0, "dataReductionStatus": "DISABLED", + "dataReductionMode": "disabled", "isAluaEnabled": False, + "naaId": self.naa(lid), + } + if paths: + view["ports"] = paths + return view + + def _ldev_id(self, raw: str) -> int: + lid = int(raw) + if lid > _MAX_LDEV: + raise _err(400, f"The value of ldevId ({lid}) is out of range.", "KART40042-E") + return lid + + @route("GET", _P + r"/ldevs/(?P\d+)") + def get_ldev(self, req, sdid, lid): + self._auth(req, sdid) + return Reply(200, self._ldev_view(self._ldev_id(lid))) + + @route("GET", _P + r"/ldevs") + def list_ldevs(self, req, sdid): + self._auth(req, sdid) + option = req.q("ldevOption") + pool = req.q("poolId") + if pool is not None and option != "dpVolume": + raise _err(400, "poolId can be specified only with ldevOption=dpVolume.", + "KART40041-E") + if option not in (None, "defined", "dpVolume"): + raise _err(400, f"fake: ldevOption={option} not implemented", "KART40041-E") + count = int(req.q("count", "100") or 100) + if not 1 <= count <= self.max_count: + raise _err(400, f"The value of count ({count}) is out of range.", "KART40042-E") + head = int(req.q("headLdevId", "0") or 0) + if head > self.last_ldev: + body = _body(f"The value of headLdevId ({head}) is out of range.", "KART40044-E") + body["errorCode"] = {"errorCode": "EX_ENLDEV"} + raise FakeError(400, body) + data = [self._ldev_view(lid) for lid in sorted(self.ldevs) + if head <= lid < head + count + and (pool is None or self.ldevs[lid]["pool"] == int(pool))] + return Reply(200, {"data": data}) + + @staticmethod + def _blocks(body: dict[str, Any]) -> int: + if "blockCapacity" in body: + return int(body["blockCapacity"]) + m = re.match(r"^(\d+)([KMGT])$", str(body.get("byteFormatCapacity", ""))) + if not m: + raise _err(400, "Specify blockCapacity or byteFormatCapacity.", "KART40041-E") + return int(m.group(1)) * _UNITS[m.group(2)] // 512 + + @route("POST", _P + r"/ldevs") + def post_ldev(self, req, sdid): + token = self._auth(req, sdid) + body = req.json() or {} + prefix = self._prefix(sdid) + if int(body.get("poolId", -1)) not in self.pools: + raise _err(400, f"The specified pool ({body.get('poolId')}) does not exist.", + "KART40045-E") + blocks = self._blocks(body) + if blocks <= 0: + raise _err(400, "The capacity is invalid.", "KART40042-E") + if ("startLdevId" in body) != ("endLdevId" in body): + raise _err(400, "Specify both startLdevId and endLdevId.", "KART40041-E") + + def run() -> list[str]: + if "ldevId" in body: + lid = int(body["ldevId"]) + if lid in self.ldevs: + raise _storage_err(f"LDEV {lid} is already defined.", "2E22", "0001") + else: + lid = self._free_ldev(int(body.get("startLdevId", 0)), + int(body.get("endLdevId", _MAX_LDEV))) + self.ldevs[lid] = {"pool": int(body["poolId"]), "blocks": blocks, "label": ""} + return [f"{prefix}/ldevs/{lid}"] + + return self._job(req, sdid, token, run) + + @route("PUT", _P + r"/ldevs/(?P\d+)") + def put_ldev(self, req, sdid, lid): + token = self._auth(req, sdid) + body = req.json() or {} + lid_i = self._ldev_id(lid) + prefix = self._prefix(sdid) + label = body.get("label") + if label is None: + raise _err(400, "fake: only label changes are implemented", "KART40041-E") + if len(label) > 32: + raise _err(400, "The label must be 32 characters or fewer.", "KART40042-E") + + def run() -> list[str]: + if lid_i not in self.ldevs: + raise _storage_err(f"LDEV {lid_i} is not defined.", "2E10", "0000") + self.ldevs[lid_i]["label"] = label + return [f"{prefix}/ldevs/{lid_i}"] + + return self._job(req, sdid, token, run) + + @route("POST", _P + r"/ldevs/(?P\d+)/actions/expand/invoke") + def expand_ldev(self, req, sdid, lid): + token = self._auth(req, sdid) + lid_i = self._ldev_id(lid) + params = (req.json() or {}).get("parameters") or {} + prefix = self._prefix(sdid) + add = int(params.get("additionalBlockCapacity", 0)) + if add <= 0: + raise _err(400, "additionalBlockCapacity must be positive.", "KART40042-E") + + def run() -> list[str]: + if lid_i not in self.ldevs: + raise _storage_err(f"LDEV {lid_i} is not defined.", "2E10", "0000") + self.ldevs[lid_i]["blocks"] += add + return [f"{prefix}/ldevs/{lid_i}"] + + return self._job(req, sdid, token, run) + + def _paths_of(self, lid: int) -> int: + return sum(1 for g in self.groups.values() for t in g["luns"].values() if t == lid) + + @route("DELETE", _P + r"/ldevs/(?P\d+)") + def delete_ldev(self, req, sdid, lid): + token = self._auth(req, sdid) + lid_i = self._ldev_id(lid) + prefix = self._prefix(sdid) + + def run() -> list[str]: + if lid_i not in self.ldevs: + raise _storage_err(f"LDEV {lid_i} is not defined.", "2E10", "0000") + if self._paths_of(lid_i): + raise _storage_err(f"LU paths are defined for LDEV {lid_i}.", "2E11", "0007") + del self.ldevs[lid_i] + return [f"{prefix}/ldevs/{lid_i}"] + + return self._job(req, sdid, token, run) + + # -- ports and host groups ------------------------------------------------------- + + def _port(self, port: str | None) -> str: + if port not in self.ports: + raise _err(404, f"The specified port ({port}) does not exist.", "KART30013-E") + return port # type: ignore[return-value] + + @route("GET", _P + r"/ports/(?P[^/]+)") + def get_port(self, req, sdid, port): + self._auth(req, sdid) + self._port(port) + info: dict[str, Any] = {"portId": port, "portType": self.ports[port], + "portAttributes": ["TAR"], "portSpeed": "AUT", + "lunSecuritySetting": True} + if self.ports[port] == "FIBRE": + info.update(loopId="EF", fabricMode=True, portConnection="PtoP", + wwn=f"50060e80{self.serial:06x}{list(self.ports).index(port):02x}") + else: + info.update(ipv4Address="192.0.2.20", tcpPort="3260") + return Reply(200, info) + + def _group_view(self, port: str, num: int) -> dict[str, Any]: + g = self.groups[(port, num)] + view: dict[str, Any] = {"hostGroupId": f"{port},{num}", "portId": port, + "hostGroupNumber": num, "hostGroupName": g["name"], + "hostMode": g["mode"], "resourceGroupId": 0} + if g["modeOptions"]: + view["hostModeOptions"] = list(g["modeOptions"]) + if self.ports[port] == "ISCSI": + view["iscsiName"] = f"iqn.1994-04.jp.co.hitachi:rsd.h8m.t.{self.serial:05d}." \ + f"{port[2:].replace('-', '')}{num:03x}" + return view + + def _existing_group(self, port: str | None, num: Any) -> tuple[str, int]: + port = self._port(port) + try: + key = (port, int(num)) + except (TypeError, ValueError) as e: + raise _err(400, "hostGroupNumber is required.", "KART40041-E") from e + if key not in self.groups: + raise _err(404, f"Host group {port},{num} does not exist.", "KART30013-E") + return key + + @route("GET", _P + r"/host-groups") + def list_groups(self, req, sdid): + self._auth(req, sdid) + port = self._port(req.q("portId")) + return Reply(200, {"data": [self._group_view(p, n) for (p, n) in sorted(self.groups) + if p == port]}) + + @route("POST", _P + r"/host-groups") + def post_group(self, req, sdid): + token = self._auth(req, sdid) + body = req.json() or {} + port = self._port(body.get("portId")) + name = str(body.get("hostGroupName") or "") + prefix = self._prefix(sdid) + limit = 32 if self.ports[port] == "ISCSI" else 64 + if not name or len(name) > limit: + raise _err(400, f"hostGroupName must be 1-{limit} characters.", "KART40042-E") + mode = body.get("hostMode", "LINUX/IRIX") + if mode not in ("LINUX/IRIX", "VMWARE_EX", "WIN_EX", "AIX", "HP-UX", "SOLARIS"): + raise _err(400, f"The hostMode ({mode}) is invalid.", "KART40042-E") + + def run() -> list[str]: + if any(p == port and g["name"] == name for (p, _), g in self.groups.items()): + raise _storage_err(f"The host group name {name} is already used on {port}.", + "2E11", "4E00") + num = body.get("hostGroupNumber") + if num is None: + num = next(n for n in range(1, 255) if (port, n) not in self.groups) + self.groups[(port, int(num))] = self._group(name, mode) + self.groups[(port, int(num))]["modeOptions"] = list(body.get("hostModeOptions", [])) + return [f"{prefix}/host-groups/{port},{num}"] + + return self._job(req, sdid, token, run) + + @route("DELETE", _P + r"/host-groups/(?P[^/,]+),(?P\d+)") + def delete_group(self, req, sdid, port, num): + token = self._auth(req, sdid) + key = self._existing_group(port, num) + prefix = self._prefix(sdid) + if key[1] == 0: + raise _err(400, "Host group 0 cannot be deleted.", "KART40042-E") + + def run() -> list[str]: + if self.groups[key]["luns"]: + raise _storage_err(f"LU paths are defined in host group {port},{num}.", + "2E11", "0007") + del self.groups[key] + return [f"{prefix}/host-groups/{port},{num}"] + + return self._job(req, sdid, token, run) + + # -- initiators -------------------------------------------------------------------- + + @route("GET", _P + r"/host-wwns") + def list_wwns(self, req, sdid): + self._auth(req, sdid) + port, num = self._existing_group(req.q("portId"), req.q("hostGroupNumber")) + g = self.groups[(port, num)] + return Reply(200, {"data": [ + {"hostWwnId": f"{port},{num},{w}", "portId": port, "hostGroupNumber": num, + "hostGroupName": g["name"], "hostWwn": w, "wwnNickname": "-"} + for w in g["wwns"]]}) + + @route("POST", _P + r"/host-wwns") + def post_wwn(self, req, sdid): + token = self._auth(req, sdid) + body = req.json() or {} + port, num = self._existing_group(body.get("portId"), body.get("hostGroupNumber")) + wwn = str(body.get("hostWwn", "")).lower() + prefix = self._prefix(sdid) + if self.ports[port] != "FIBRE": + raise _err(400, f"Port {port} is not a Fibre Channel port.", "KART40042-E") + if not re.match(r"^[0-9a-f]{16}$", wwn): + raise _err(400, f"The hostWwn ({wwn}) is invalid.", "KART40042-E") + + def run() -> list[str]: + for (p, n), g in self.groups.items(): + if p == port and wwn in g["wwns"]: + raise _storage_err( + f"The WWN {wwn} is already registered in host group {p},{n}.", + "B957", "4180") + self.groups[(port, num)]["wwns"].append(wwn) + return [f"{prefix}/host-wwns/{port},{num},{wwn}"] + + return self._job(req, sdid, token, run) + + @route("GET", _P + r"/host-iscsis") + def list_iscsis(self, req, sdid): + self._auth(req, sdid) + port, num = self._existing_group(req.q("portId"), req.q("hostGroupNumber")) + g = self.groups[(port, num)] + return Reply(200, {"data": [ + {"hostIscsiId": f"{port},{num},{n}", "portId": port, "hostGroupNumber": num, + "hostGroupName": g["name"], "iscsiName": n, "iscsiNickname": "-"} + for n in g["iscsis"]]}) + + @route("POST", _P + r"/host-iscsis") + def post_iscsi(self, req, sdid): + token = self._auth(req, sdid) + body = req.json() or {} + port, num = self._existing_group(body.get("portId"), body.get("hostGroupNumber")) + name = str(body.get("iscsiName", "")) + prefix = self._prefix(sdid) + if self.ports[port] != "ISCSI": + raise _err(400, f"Port {port} is not an iSCSI port.", "KART40042-E") + if not name.startswith(("iqn.", "eui.")): + raise _err(400, f"The iscsiName ({name}) is invalid.", "KART40042-E") + + def run() -> list[str]: + g = self.groups[(port, num)] + # The same initiator may log in to several targets of a port; only + # a duplicate within one target is refused. + if name.lower() in (n.lower() for n in g["iscsis"]): + raise _storage_err(f"The iSCSI name {name} is already registered.", + "B957", "4185") + g["iscsis"].append(name) + return [f"{prefix}/host-iscsis/{port},{num},{name}"] + + return self._job(req, sdid, token, run) + + # -- LU paths --------------------------------------------------------------------- + + @route("GET", _P + r"/luns") + def list_luns(self, req, sdid): + self._auth(req, sdid) + port, num = self._existing_group(req.q("portId"), req.q("hostGroupNumber")) + g = self.groups[(port, num)] + return Reply(200, {"data": [ + {"lunId": f"{port},{num},{lun}", "portId": port, "hostGroupNumber": num, + "hostMode": g["mode"], "lun": lun, "ldevId": lid, "isCommandDevice": False, + "luHostReserve": {"openSystem": False, "persistent": False, + "pgrKey": False, "mainframe": False, "acaReserve": False}} + for lun, lid in sorted(g["luns"].items())]}) + + @route("POST", _P + r"/luns") + def post_lun(self, req, sdid): + token = self._auth(req, sdid) + body = req.json() or {} + port, num = self._existing_group(body.get("portId"), body.get("hostGroupNumber")) + lid = int(body.get("ldevId", -1)) + prefix = self._prefix(sdid) + lun = body.get("lun") + if lun is not None and not 0 <= int(lun) <= 2047: + raise _err(400, f"The value of lun ({lun}) is out of range.", "KART40042-E") + + def run() -> list[str]: + g = self.groups[(port, num)] + if lid not in self.ldevs: + raise _storage_err(f"LDEV {lid} is not defined.", "2E10", "0000") + if lid in g["luns"].values(): + raise _storage_err(f"LDEV {lid} is already mapped in host group " + f"{port},{num}.", "B958", "015A") + n = int(lun) if lun is not None else next( + i for i in range(2048) if i not in g["luns"]) + if n in g["luns"]: + raise _storage_err(f"LUN {n} is already in use in host group " + f"{port},{num}.", "B958", "0947") + g["luns"][n] = lid + return [f"{prefix}/luns/{port},{num},{n}"] + + return self._job(req, sdid, token, run) + + @route("DELETE", _P + r"/luns/(?P[^/,]+),(?P\d+),(?P\d+)") + def delete_lun(self, req, sdid, port, num, lun): + token = self._auth(req, sdid) + key = self._existing_group(port, num) + prefix = self._prefix(sdid) + if int(lun) not in self.groups[key]["luns"]: + raise _err(404, f"LU path {port},{num},{lun} does not exist.", "KART30013-E") + + def run() -> list[str]: + self.groups[key]["luns"].pop(int(lun), None) + return [f"{prefix}/luns/{port},{num},{lun}"] + + return self._job(req, sdid, token, run) + + +def contract_setup(**options: Any) -> ContractSetup: + fake = FakeHitachiVsp(username="sbadmin", password="vsp-secret") + opts: dict[str, Any] = {"pool_id": 0, "ports": ["CL1-A", "CL2-A", "CL1-B", "CL2-B"]} + opts.update(options) + return ContractSetup( + fake=fake, + config=make_config("hitachi-vsp", "https://vsp.fake.example", **opts), + credential=Credential({"username": SecretStr("sbadmin"), + "password": SecretStr("vsp-secret")}), + bad_credential=Credential({"username": SecretStr("sbadmin"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_ldev(name, size), + ) diff --git a/simplyblock_core/storage_arrays/testing/hpe_nimble.py b/simplyblock_core/storage_arrays/testing/hpe_nimble.py new file mode 100644 index 0000000000..e03cbd5982 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/hpe_nimble.py @@ -0,0 +1,425 @@ +"""Fake HPE Alletra 5000/6000 / Nimble, NimbleOS REST v1 (subset the adapter uses). + +Quirks reproduced: tokens from ``POST /v1/tokens`` checked on ``X-Auth-Token`` +and released by token *id*; ``{"data": ...}`` envelopes on requests and +responses; ``{"messages": [{"code": "SM_...", ...}]}`` errors; 42-hex object +ids; sizes in MiB; volumes that must be offline before deletion and cannot be +offlined or deleted while ACRs exist; FC WWPNs in colon form; and LUN 0 for +every iSCSI ACR with volume-scoped targets. + +Modelled on the NimbleOS SDK and Cinder: error bodies carry a generic +``SM_http_`` message followed by the specific one (``SM_eexist``, +``SM_http_not_found``, ``SM_vol_size_decreased``, ...); offlining an offline +volume is refused with "Object is already in requested state"; and with +``async_ops`` a mutating request answers 202 + ``SM_async_job_id`` and the +job is polled on ``GET /v1/jobs/`` until ``state`` is ``done``. The +specific codes for ACR/online refusals (``SM_vol_online``, +``SM_vol_has_acl``, ``SM_vol_has_connections``, ``SM_igrp_has_acl``) are +the fake's own. +""" +from __future__ import annotations + +import re +import secrets +from typing import Any + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +_ID = r"(?P(?!detail$)[^/]+)" +_ID_RE = re.compile(r"^[0-9a-f]{42}$") +_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.:\-]{0,214}$") +_WWPN_RE = re.compile(r"^([0-9a-f]{2}:){7}[0-9a-f]{2}$") +_NG_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.:\-]{0,63}$") + + +_HTTP_CODE = {400: "SM_http_bad_request", 401: "SM_http_unauthorized", + 404: "SM_http_not_found", 409: "SM_http_conflict"} + + +def _err(status: int, code: str, text: str) -> FakeError: + messages = [{"code": code, "severity": "error", "text": text}] + generic = _HTTP_CODE.get(status) + if generic and generic != code: + messages.insert(0, {"code": generic, "severity": "error", "text": text}) + return FakeError(status, {"messages": messages}) + + +class FakeHpeNimble(FakeArray): + vendor = "hpe-nimble" + + def __init__(self, username: str = "admin", password: str = "admin", + volume_scoped_targets: bool = True): + super().__init__() + self.username = username + self.password = password + self.volume_scoped_targets = volume_scoped_targets + #: "METHOD kind" pairs (kind = volume, initiator_group, initiator, acr) + #: that answer 202 + an async job instead of the object. + self.async_ops: set[str] = set() + #: polls a job reports ``running`` before ``done`` + self.job_polls = 2 + self.jobs: dict[str, int] = {} # job id -> polls left + self.tokens: dict[str, str] = {} # token id -> session_token + self.pools = {self._new_id(0x0a): "default", self._new_id(0x0a): "fast"} + self.perfpolicies = {self._new_id(0x03): "default", self._new_id(0x03): "Oracle OLTP"} + self.volumes: dict[str, dict[str, Any]] = {} # id -> volume + self.groups: dict[str, dict[str, Any]] = {} # id -> initiator group + self.initiators: dict[str, dict[str, Any]] = {} + self.acrs: dict[str, dict[str, Any]] = {} + + def _new_id(self, kind: int) -> str: + # NimbleOS ids: 42 hex chars, group-unique prefix + object type + counter. + return f"06{secrets.token_hex(9)}{kind:02x}{self.next_id():020x}"[:42] + + # -- fake controls ------------------------------------------------------ + + def expire_sessions(self) -> None: + self.tokens.clear() + + def set_credential(self, **fields: str) -> None: + self.username = fields.get("username", self.username) + self.password = fields["password"] + + def add_volume(self, name: str, size: int, pool_id: str | None = None, + perfpolicy_id: str | None = None) -> str: + vid = self._new_id(0x2a) + pool_id = pool_id or next(iter(self.pools)) + perfpolicy_id = perfpolicy_id or next(iter(self.perfpolicies)) + self.volumes[vid] = { + "id": vid, "name": name, "size": -(-size // (1 << 20)), + "serial_number": secrets.token_hex(8) + "6c9ce900" + secrets.token_hex(4), + "target_name": f"iqn.2007-11.com.nimblestorage:{name}-v{secrets.token_hex(8)}", + "online": True, "vol_state": "online", "pool_id": pool_id, + "pool_name": self.pools[pool_id], "perfpolicy_id": perfpolicy_id, + "perfpolicy_name": self.perfpolicies[perfpolicy_id], "access_control_records": [], + } + return vid + + # -- envelope helpers ----------------------------------------------------- + + def _auth(self, req: Request) -> None: + if req.headers.get("X-Auth-Token") not in self.tokens.values(): + raise _err(401, "SM_http_unauthorized", "Unauthorized: session token invalid") + + @staticmethod + def _body(req: Request) -> dict[str, Any]: + body = req.json() + if not isinstance(body, dict) or not isinstance(body.get("data"), dict): + raise _err(400, "SM_missing_arg", "Missing data envelope") + return body["data"] + + @staticmethod + def _page(items: list[dict[str, Any]]) -> Reply: + return Reply(200, {"startRow": 0, "endRow": len(items), "totalRows": len(items), + "data": items}) + + @staticmethod + def _filter(req: Request, items: list[dict[str, Any]]) -> list[dict[str, Any]]: + out = items + for key, values in req.query.items(): + if key in ("startRow", "endRow", "pageSize", "fields", "sortBy"): + continue + out = [i for i in out if str(i.get(key, "")).lower() == values[0].lower()] + return out + + def _get(self, table: dict[str, dict[str, Any]], obj_id: str, what: str) -> dict[str, Any]: + if not _ID_RE.match(obj_id): + raise _err(400, "SM_invalid_arg", f"Invalid {what} id: {obj_id}") + if obj_id not in table: + raise _err(404, "SM_http_not_found", f"Object not found: {what} {obj_id}") + return table[obj_id] + + def _done(self, op: str, status: int, data: Any) -> Reply: + """The reply for a completed mutation, or 202 + a job for ``async_ops``.""" + if op not in self.async_ops: + return Reply(status, {"data": data} if data is not None else {}) + jid = self._new_id(0x29) + self.jobs[jid] = self.job_polls + return Reply(202, {"messages": [{"code": "SM_async_job_id", "severity": "info", + "text": f"Job {jid} is running", + "arguments": {"job_id": jid}}]}) + + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return list(self.volumes) + + def lio_all_iqns(self) -> list[str]: + return sorted({i["iqn"].lower() for i in self.initiators.values() if i.get("iqn")}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + gids = {g["id"] for g in self.groups.values() if host_id in (g["id"], g["name"])} + return [i["iqn"].lower() for i in self.initiators.values() + if i["initiator_group_id"] in gids and i.get("iqn")] + + @route("GET", "/v1/jobs/" + _ID) + def get_job(self, req, obj_id): + self._auth(req) + if obj_id not in self.jobs: + raise _err(404, "SM_http_not_found", f"Object not found: job {obj_id}") + left = self.jobs[obj_id] + self.jobs[obj_id] = max(0, left - 1) + return Reply(200, {"data": {"id": obj_id, "state": "running" if left else "done", + "result": "" if left else "success"}}) + + # -- auth / info ---------------------------------------------------------- + + @route("POST", "/v1/tokens") + def login(self, req): + data = self._body(req) + if data.get("username") != self.username or data.get("password") != self.password: + raise _err(401, "SM_http_unauthorized", "Unauthorized: invalid credentials") + tid, token = self._new_id(0x1f), secrets.token_hex(16) + self.tokens[tid] = token + return Reply(201, {"data": {"id": tid, "session_token": token, + "username": self.username, "app_name": "", + "source_ip": "192.0.2.1"}}) + + @route("DELETE", "/v1/tokens/" + _ID) + def logout(self, req, obj_id): + self._auth(req) + self._get(self.tokens, obj_id, "token") # type: ignore[arg-type] + del self.tokens[obj_id] + return Reply(200, {}) + + @route("GET", "/v1/arrays/detail") + def arrays(self, req): + self._auth(req) + return self._page([{"id": "0906b8bc6e4a3c2a7f000000000000000000000001", + "name": "fake-af40", "model": "AF40", "serial": "AF-123456", + "version": "6.1.2.100-1010", "status": "reachable"}]) + + @route("GET", "/v1/pools/detail") + def pools_detail(self, req): + self._auth(req) + return self._page(self._filter(req, [{"id": k, "name": v} + for k, v in self.pools.items()])) + + @route("GET", "/v1/performance_policies/detail") + def perfpolicies_detail(self, req): + self._auth(req) + return self._page(self._filter(req, [{"id": k, "name": v} + for k, v in self.perfpolicies.items()])) + + # -- volumes ------------------------------------------------------------- + + @route("POST", "/v1/volumes") + def post_volumes(self, req): + self._auth(req) + data = self._body(req) + name, size = data.get("name"), data.get("size") + if not isinstance(name, str) or not _NAME_RE.match(name): + raise _err(400, "SM_invalid_arg", "Invalid argument: name") + if not isinstance(size, int) or size < 1: + raise _err(400, "SM_invalid_arg", "Invalid argument: size (MiB)") + if any(v["name"] == name for v in self.volumes.values()): + raise _err(409, "SM_eexist", f"Object exists: volume {name}") + pool_id, pp_id = data.get("pool_id"), data.get("perfpolicy_id") + if pool_id is not None and pool_id not in self.pools: + raise _err(404, "SM_http_not_found", f"Object not found: pool {pool_id}") + if pp_id is not None and pp_id not in self.perfpolicies: + raise _err(404, "SM_http_not_found", f"Object not found: perfpolicy {pp_id}") + vid = self.add_volume(name, size << 20, pool_id, pp_id) + return self._done("POST volume", 201, self.volumes[vid]) + + @route("GET", "/v1/volumes/detail") + def volumes_detail(self, req): + self._auth(req) + return self._page(self._filter(req, list(self.volumes.values()))) + + @route("GET", "/v1/volumes/" + _ID) + def get_volume(self, req, obj_id): + self._auth(req) + return Reply(200, {"data": self._get(self.volumes, obj_id, "volume")}) + + @route("PUT", "/v1/volumes/" + _ID) + def put_volume(self, req, obj_id): + self._auth(req) + vol = self._get(self.volumes, obj_id, "volume") + data = self._body(req) + if "size" in data: + size = data["size"] + if not isinstance(size, int) or size < 1: + raise _err(400, "SM_invalid_arg", "Invalid argument: size (MiB)") + if size < vol["size"] and not data.get("force"): + raise _err(400, "SM_vol_size_decreased", + "the operation will decrease the size of the volume. " + "use force option to proceed.") + vol["size"] = size + if "online" in data: + if bool(data["online"]) == vol["online"]: + raise _err(400, "SM_ealready", "Object is already in requested state") + if not data["online"] and self._vol_acrs(obj_id) and not data.get("force"): + raise _err(409, "SM_vol_has_connections", + "Volume has access control records; cannot offline") + vol["online"] = bool(data["online"]) + vol["vol_state"] = "online" if vol["online"] else "offline" + return self._done("PUT volume", 200, vol) + + @route("DELETE", "/v1/volumes/" + _ID) + def delete_volume(self, req, obj_id): + self._auth(req) + vol = self._get(self.volumes, obj_id, "volume") + if vol["online"]: + raise _err(409, "SM_vol_online", "Volume must be offline before it can be deleted") + if self._vol_acrs(obj_id): + raise _err(409, "SM_vol_has_acl", "Volume has access control records") + del self.volumes[obj_id] + return self._done("DELETE volume", 200, None) + + # -- initiator groups / initiators ---------------------------------------------- + + def _group_view(self, g: dict[str, Any]) -> dict[str, Any]: + members = [i for i in self.initiators.values() if i["initiator_group_id"] == g["id"]] + view = dict(g) + if g["access_protocol"] == "iscsi": + view["iscsi_initiators"] = [{"id": i["id"], "label": i["label"], "iqn": i["iqn"], + "ip_address": "*"} for i in members] + view["fc_initiators"] = None + else: + view["fc_initiators"] = [{"wwpn": i["wwpn"], "alias": ""} for i in members] + view["iscsi_initiators"] = None + return view + + @route("GET", "/v1/initiator_groups/detail") + def groups_detail(self, req): + self._auth(req) + return self._page(self._filter(req, [self._group_view(g) + for g in self.groups.values()])) + + @route("GET", "/v1/initiator_groups/" + _ID) + def get_group(self, req, obj_id): + self._auth(req) + return Reply(200, {"data": self._group_view(self._get(self.groups, obj_id, "group"))}) + + @route("POST", "/v1/initiator_groups") + def post_group(self, req): + self._auth(req) + data = self._body(req) + name, proto = data.get("name"), data.get("access_protocol") + if not isinstance(name, str) or not _NG_NAME_RE.match(name): + raise _err(400, "SM_invalid_arg", "Invalid argument: name") + if proto not in ("iscsi", "fc"): + raise _err(400, "SM_invalid_arg", "Invalid argument: access_protocol") + if any(g["name"] == name for g in self.groups.values()): + raise _err(409, "SM_eexist", f"Object exists: initiator group {name}") + gid = self._new_id(0x02) + self.groups[gid] = {"id": gid, "name": name, "access_protocol": proto, + "description": data.get("description", ""), + "target_subnets": [], "host_type": "auto"} + return self._done("POST initiator_group", 201, self._group_view(self.groups[gid])) + + @route("DELETE", "/v1/initiator_groups/" + _ID) + def delete_group(self, req, obj_id): + self._auth(req) + self._get(self.groups, obj_id, "group") + if any(a["initiator_group_id"] == obj_id for a in self.acrs.values()): + raise _err(409, "SM_igrp_has_acl", + "Initiator group is in use by access control records") + for iid in [i for i, v in self.initiators.items() if v["initiator_group_id"] == obj_id]: + del self.initiators[iid] + del self.groups[obj_id] + return self._done("DELETE initiator_group", 200, None) + + @route("GET", "/v1/initiators/detail") + def initiators_detail(self, req): + self._auth(req) + return self._page(self._filter(req, list(self.initiators.values()))) + + @route("POST", "/v1/initiators") + def post_initiator(self, req): + self._auth(req) + data = self._body(req) + gid, proto = data.get("initiator_group_id"), data.get("access_protocol") + group = self._get(self.groups, str(gid), "group") + if proto != group["access_protocol"]: + raise _err(400, "SM_invalid_arg", "access_protocol does not match initiator group") + item = {"id": self._new_id(0x04), "access_protocol": proto, + "initiator_group_id": gid, "initiator_group_name": group["name"], + "label": "", "iqn": "", "ip_address": "", "wwpn": "", "alias": ""} + if proto == "iscsi": + iqn, label = data.get("iqn"), data.get("label") + if not iqn or not label or len(label) > 64: + raise _err(400, "SM_missing_arg", "iqn and label are required") + if any(i["iqn"].lower() == iqn.lower() for i in self.initiators.values()): + raise _err(409, "SM_eexist", f"Object exists: initiator {iqn}") + item.update(iqn=iqn, label=label, ip_address=data.get("ip_address", "*")) + else: + wwpn = str(data.get("wwpn", "")) + if not _WWPN_RE.match(wwpn): + raise _err(400, "SM_invalid_arg", "Invalid argument: wwpn (colon form)") + if any(i["wwpn"] == wwpn for i in self.initiators.values()): + raise _err(409, "SM_eexist", f"Object exists: initiator {wwpn}") + item.update(wwpn=wwpn, alias=data.get("alias", "")) + self.initiators[item["id"]] = item + return self._done("POST initiator", 201, item) + + # -- access control records ------------------------------------------------ + + def _vol_acrs(self, vol_id: str) -> list[dict[str, Any]]: + return [a for a in self.acrs.values() if a["vol_id"] == vol_id] + + @route("GET", "/v1/access_control_records/detail") + def acrs_detail(self, req): + self._auth(req) + return self._page(self._filter(req, list(self.acrs.values()))) + + @route("POST", "/v1/access_control_records") + def post_acr(self, req): + self._auth(req) + data = self._body(req) + vol = self._get(self.volumes, str(data.get("vol_id")), "volume") + group = self._get(self.groups, str(data.get("initiator_group_id")), "group") + if data.get("apply_to", "both") not in ("volume", "snapshot", "both", "pe", "vvol_volume", + "vvol_snapshot"): + raise _err(400, "SM_invalid_arg", "Invalid argument: apply_to") + if any(a["initiator_group_id"] == group["id"] for a in self._vol_acrs(vol["id"])): + raise _err(409, "SM_eexist", "Object exists: access control record") + if group["access_protocol"] == "iscsi" and self.volume_scoped_targets: + lun = 0 + else: + used = {a["lun"] for a in self.acrs.values() + if a["initiator_group_id"] == group["id"]} + lun = next(n for n in range(2048) if n not in used) + aid = self._new_id(0x0d) + self.acrs[aid] = {"id": aid, "vol_id": vol["id"], "vol_name": vol["name"], + "initiator_group_id": group["id"], + "initiator_group_name": group["name"], + "apply_to": data.get("apply_to", "both"), "lun": lun, + "chap_user_id": "", "pe_id": "", "snap_id": ""} + return self._done("POST acr", 201, self.acrs[aid]) + + @route("DELETE", "/v1/access_control_records/" + _ID) + def delete_acr(self, req, obj_id): + self._auth(req) + self._get(self.acrs, obj_id, "access control record") + del self.acrs[obj_id] + return self._done("DELETE acr", 200, None) + + +def contract_setup() -> ContractSetup: + fake = FakeHpeNimble(username="admin", password="nimble-pw") + return ContractSetup( + fake=fake, + config=make_config("hpe-nimble", "https://nimble.fake.example:5392"), + credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("nimble-pw")}), + bad_credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_volume(name, size), + ) diff --git a/simplyblock_core/storage_arrays/testing/hpe_wsapi.py b/simplyblock_core/storage_arrays/testing/hpe_wsapi.py new file mode 100644 index 0000000000..36d8c559ff --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/hpe_wsapi.py @@ -0,0 +1,351 @@ +"""Fake HPE WSAPI (Alletra MP / Primera / 3PAR), the subset the adapter uses. + +Quirks reproduced: session keys created with ``POST /api/v1/credentials`` and +checked on ``X-HP3PAR-WSAPI-SessionKey``; ``{"code", "desc"}`` error bodies; +volumes addressed by name with sizes in MiB rounded up to 256 MiB; GROW +taking an increment; host requests using ``iSCSINames``/``FCWWNs`` while +responses report ``iSCSIPaths``/``FCPaths``; VLUN creation answering 201 +with an empty body and a ``Location`` header; and VLUN listings that contain +the template plus one active VLUN per host path. + +Error codes and statuses follow python-3parclient's own WSAPI mock +(``test/HPE3ParMockServer_flask.py``): 5 INV_USER_PASS, 12 INV_INPUT, 15 +NON_EXISTENT_CPG, 16 EXISTENT_HOST, 17 NON_EXISTENT_HOST, 19 +NON_EXISTENT_VLUN, 22 EXISTENT_VOL, 23 NON_EXISTENT_VOL, 26 EXPORTED_VLUN, +39 INV_INPUT_WRONG_TYPE, 40 INV_INPUT_MISSING_REQUIRED, 43 +INV_INPUT_EXCEEDS_RANGE (create below 256 MiB), 44 INV_INPUT_PARAM_CONFLICT, +57 INV_INPUT_EXCEEDS_LENGTH, 73 EXISTENT_PATH, 152 INV_INPUT_VV_GROW_SIZE, +153 VV_NEW_SIZE_EXCEED_CPG_LIMIT (a *403* that is not an auth failure), plus +34 "resource in use" (409) from HPE flowkit. The session-expiry answer (401 +code 6) is a guess. +""" +from __future__ import annotations + +import re +import secrets +from typing import Any +from urllib.parse import unquote + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +_A = "/api/v1" +_UNIT_MIB = 256 +_MIN_MIB = 256 +_MAX_NAME = 31 +_QUERY_RE = re.compile(r'^"?\s*volumeName\s+EQ\s+(?P\S+?)\s*"?$') + + +def _err(status: int, code: int, desc: str) -> FakeError: + return FakeError(status, {"code": code, "desc": desc}) + + +class FakeHpeWsapi(FakeArray): + vendor = "hpe-wsapi" + + def __init__(self, username: str = "3paradm", password: str = "3pardata", + cpgs: tuple[str, ...] = ("SSD_r6",), paths_per_host: int = 2, + cpg_limit_mib: int | None = None): + super().__init__() + self.username = username + self.password = password + self.cpgs = set(cpgs) + #: active VLUN entries listed per mapping (one per host port path) + self.paths_per_host = paths_per_host + #: growth limit per volume; exceeding it answers 403 code 153 + self.cpg_limit_mib = cpg_limit_mib + self.sessions: set[str] = set() + self.volumes: dict[str, dict[str, Any]] = {} # name -> volume + self.hosts: dict[str, dict[str, Any]] = {} # name -> host + self.vluns: list[dict[str, Any]] = [] # templates {volumeName, hostname, lun} + self.created_bodies: list[dict[str, Any]] = [] + + # -- fake controls ------------------------------------------------------ + + def expire_sessions(self) -> None: + self.sessions.clear() + + def set_credential(self, **fields: str) -> None: + self.username = fields.get("username", self.username) + self.password = fields["password"] + + def add_volume(self, name: str, size: int) -> str: + mib = -(-size // (1 << 20)) + mib = -(-mib // _UNIT_MIB) * _UNIT_MIB + self.volumes[name] = { + "id": self.next_id(), "name": name, "sizeMiB": mib, + "wwn": "60002AC" + secrets.token_hex(13).upper()[:25], + "provisioningType": 2, "userCPG": next(iter(self.cpgs)), "state": 1, + "uuid": secrets.token_hex(16), + } + return name + + # -- auth --------------------------------------------------------------- + + def _auth(self, req: Request) -> None: + if req.headers.get("X-HP3PAR-WSAPI-SessionKey") not in self.sessions: + raise _err(401, 6, "invalid session key") + + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return list(self.volumes) + + def lio_all_iqns(self) -> list[str]: + return sorted({p["name"].lower() for h in self.hosts.values() for p in h.get("iSCSIPaths", [])}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + host = self.hosts.get(host_id) or {} + return [p["name"].lower() for p in host.get("iSCSIPaths", [])] + + @route("POST", _A + "/credentials") + def login(self, req): + body = req.json() or {} + if body.get("user") != self.username or body.get("password") != self.password: + raise _err(403, 5, "invalid username or password") + key = "0-" + secrets.token_hex(16) + "-" + secrets.token_hex(4) + self.sessions.add(key) + return Reply(201, {"key": key}) + + @route("DELETE", _A + "/credentials/(?P[^/]+)") + def logout(self, req, key): + key = unquote(key) + if key not in self.sessions: + raise _err(404, 6, "invalid session key") + self.sessions.discard(key) + return Reply(200) + + @route("GET", _A + "/system") + def system(self, req): + self._auth(req) + return Reply(200, {"id": 412345, "name": "fake-b10000", "systemVersion": "10.4.0.123", + "model": "HPE Alletra Storage MP B10000", + "serialNumber": "CZ12345678", "IPv4Addr": "192.0.2.10"}) + + # -- volumes ------------------------------------------------------------- + + def _vol(self, name: str) -> dict[str, Any]: + name = unquote(name) + if name not in self.volumes: + raise _err(404, 23, "volume does not exist") + return self.volumes[name] + + @route("POST", _A + "/volumes") + def post_volumes(self, req): + self._auth(req) + body = req.json() or {} + self.created_bodies.append(body) + name, cpg, size = body.get("name"), body.get("cpg"), body.get("sizeMiB") + if not name or not cpg or not isinstance(size, int): + raise _err(400, 40, "missing required field") + if len(name) > _MAX_NAME: + raise _err(400, 57, "invalid input: string length exceeds limit") + if cpg not in self.cpgs: + raise _err(404, 15, "CPG does not exist") + if name in self.volumes: + raise _err(409, 22, "volume exists") + if size < _MIN_MIB: + raise _err(400, 43, "Minimum volume size is 256 MiB") + self.add_volume(name, size << 20) + return Reply(201, None, {"Location": f"{_A}/volumes/{name}"}) + + @route("GET", _A + "/volumes/(?P[^/]+)") + def get_volume(self, req, name): + self._auth(req) + return Reply(200, self._vol(name)) + + @route("PUT", _A + "/volumes/(?P[^/]+)") + def put_volume(self, req, name): + self._auth(req) + vol = self._vol(name) + body = req.json() or {} + if body.get("action") == 3: + delta = body.get("sizeMiB") + if not isinstance(delta, int) or delta <= 0: + raise _err(400, 152, "invalid grow size") + new = -(-(vol["sizeMiB"] + delta) // _UNIT_MIB) * _UNIT_MIB + if self.cpg_limit_mib is not None and new > self.cpg_limit_mib: + raise _err(403, 153, "new volume size exceeds CPG limit") + vol["sizeMiB"] = new + return Reply(200) + raise _err(400, 12, "invalid input: unsupported action") + + @route("DELETE", _A + "/volumes/(?P[^/]+)") + def delete_volume(self, req, name): + self._auth(req) + vol = self._vol(name) + if any(v["volumeName"] == vol["name"] for v in self.vluns): + raise _err(409, 34, "volume is exported (has VLUNs)") + del self.volumes[vol["name"]] + return Reply(200) + + # -- hosts ---------------------------------------------------------------- + + def _host(self, name: str) -> dict[str, Any]: + name = unquote(name) + if name not in self.hosts: + raise _err(404, 17, "host does not exist") + return self.hosts[name] + + def _claim(self, host: str, body: dict[str, Any]) -> None: + iqns = [str(i) for i in body.get("iSCSINames") or []] + wwns = [str(w).upper().replace(":", "") for w in body.get("FCWWNs") or []] + for w in wwns: + if not re.fullmatch(r"[0-9A-F]{16}", w): + raise _err(400, 39, "invalid WWN") + for other in self.hosts.values(): + if other["name"] == host: + continue + taken = ({p["name"].lower() for p in other["iSCSIPaths"]} & {i.lower() for i in iqns} + or {p["wwn"] for p in other["FCPaths"]} & set(wwns)) + if taken: + raise _err(409, 73, f"path {min(taken)} is already used by " + f"host {other['name']}") + h = self.hosts[host] + for i in iqns: + if i.lower() not in {p["name"].lower() for p in h["iSCSIPaths"]}: + h["iSCSIPaths"].append({"name": i, "IPAddr": "0.0.0.0", "hostSpeed": 0}) + for w in wwns: + if w not in {p["wwn"] for p in h["FCPaths"]}: + h["FCPaths"].append({"wwn": w, "hostSpeed": 0}) + + @route("POST", _A + "/hosts") + def post_hosts(self, req): + self._auth(req) + body = req.json() or {} + name = body.get("name") + if not name: + raise _err(400, 40, "missing required field") + if len(name) > _MAX_NAME: + raise _err(400, 57, "invalid input: string length exceeds limit") + if name in self.hosts: + raise _err(409, 16, "host exists") + if body.get("iSCSINames") and body.get("FCWWNs"): + raise _err(400, 44, "invalid input: iSCSINames and FCWWNs are exclusive") + self.hosts[name] = {"id": self.next_id(), "name": name, + "persona": body.get("persona", 2), + "iSCSIPaths": [], "FCPaths": [], "descriptors": {}} + try: + self._claim(name, body) + except FakeError: + del self.hosts[name] + raise + return Reply(201, None, {"Location": f"{_A}/hosts/{name}"}) + + @route("GET", _A + "/hosts/(?P[^/]+)") + def get_host(self, req, name): + self._auth(req) + return Reply(200, self._host(name)) + + @route("PUT", _A + "/hosts/(?P[^/]+)") + def put_host(self, req, name): + self._auth(req) + host = self._host(name) + body = req.json() or {} + if body.get("pathOperation") != 1: + raise _err(400, 46, "invalid input: unsupported pathOperation") + if body.get("iSCSINames") and body.get("FCWWNs"): + raise _err(400, 44, "invalid input: iSCSINames and FCWWNs are exclusive") + self._claim(host["name"], body) + return Reply(200) + + @route("DELETE", _A + "/hosts/(?P[^/]+)") + def delete_host(self, req, name): + self._auth(req) + host = self._host(name) + if any(v["hostname"] == host["name"] for v in self.vluns): + raise _err(409, 26, "host has exported VLUNs") + del self.hosts[host["name"]] + return Reply(200) + + # -- VLUNs ------------------------------------------------------------------ + + def _vlun_members(self, vol: str | None) -> list[dict[str, Any]]: + out = [] + for t in self.vluns: + if vol is not None and t["volumeName"] != vol: + continue + v = self.volumes[t["volumeName"]] + base = {"lun": t["lun"], "volumeName": t["volumeName"], "hostname": t["hostname"], + "volumeWWN": v["wwn"], "type": 4} + out.append({**base, "active": False}) + for port in range(self.paths_per_host): + out.append({**base, "active": True, "remoteName": f"path{port}", + "portPos": {"node": port, "slot": 2, "cardPort": 1}}) + return out + + @route("GET", _A + "/vluns") + def get_vluns(self, req): + self._auth(req) + query = req.q("query") + vol = None + if query is not None: + m = _QUERY_RE.match(query) + if not m: + raise _err(400, 92, "invalid query") + vol = m["vol"] + members = self._vlun_members(vol) + return Reply(200, {"total": len(members), "members": members}) + + @route("POST", _A + "/vluns") + def post_vluns(self, req): + self._auth(req) + body = req.json() or {} + vol, host = body.get("volumeName"), body.get("hostname") + if vol not in self.volumes: + raise _err(404, 23, "volume does not exist") + if host not in self.hosts: + raise _err(404, 17, "host does not exist") + if any(t["volumeName"] == vol and t["hostname"] == host for t in self.vluns): + # The real code for a duplicate VLUN is not in vendor source. + raise _err(409, 0, "VLUN exists") + used = {t["lun"] for t in self.vluns if t["hostname"] == host} + start = int(body.get("lun", 0)) + if body.get("autoLun"): + lun = next(n for n in range(start, 16384) if n not in used) + elif start in used: + raise _err(409, 0, "LUN already in use") + else: + lun = start + self.vluns.append({"volumeName": vol, "hostname": host, "lun": lun}) + return Reply(201, None, {"Location": f"{_A}/vluns/{vol},{lun},{host}"}) + + @route("DELETE", _A + "/vluns/(?P[^/]+)") + def delete_vlun(self, req, spec): + self._auth(req) + parts = unquote(spec).split(",") + if len(parts) != 3 or not parts[1].isdigit(): + raise _err(400, 12, "invalid VLUN id") + vol, lun, host = parts[0], int(parts[1]), parts[2] + for t in self.vluns: + if t["volumeName"] == vol and t["lun"] == lun and t["hostname"] == host: + self.vluns.remove(t) + return Reply(200) + raise _err(404, 19, "VLUN does not exist") + + +def contract_setup() -> ContractSetup: + fake = FakeHpeWsapi(username="3paradm", password="wsapi-pw") + return ContractSetup( + fake=fake, + config=make_config("hpe-wsapi", "https://wsapi.fake.example:443", cpg="SSD_r6"), + credential=Credential({"username": SecretStr("3paradm"), + "password": SecretStr("wsapi-pw")}), + bad_credential=Credential({"username": SecretStr("3paradm"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_volume(name, size), + ) diff --git a/simplyblock_core/storage_arrays/testing/huawei_dorado.py b/simplyblock_core/storage_arrays/testing/huawei_dorado.py new file mode 100644 index 0000000000..0917185695 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/huawei_dorado.py @@ -0,0 +1,610 @@ +"""Fake Huawei OceanStor Dorado DeviceManager REST API (the subset the adapter uses). + +Quirks reproduced: + +* every answer is HTTP 200 with ``{"data": ..., "error": {"code", "description"}}``; + failures only show in ``error.code``; a query matching nothing omits ``data``; +* login at ``/deviceManager/rest/xx/sessions`` returns ``deviceid`` + + ``iBaseToken``; later calls must use ``/deviceManager/rest//`` and + the ``iBaseToken`` header, else ``-401``; wrong password is 1077987870; +* every value is a string; LUN ``CAPACITY`` is in 512-byte sectors; +* initiators are free-standing objects bound to a host via ``PARENTID``; + ``ignore_parent_filter`` makes the ``?PARENTID=`` list filter a no-op; +* name lookups are ``?filter=NAME::&range=[0-100]`` (exact match); +* mapping is host group + LUN group + mapping view; a host can be in one host + group only; host LUN ids are assigned when a LUN becomes visible to a host + and reported inside the ``ASSOCIATEMETADATA`` JSON string. + +Codes marked "unverified" below are stand-ins: the real array refuses those +operations, but we do not know with which code. The adapter must not depend +on them. +""" +from __future__ import annotations + +import json +import secrets +from typing import Any + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +_D = r"/deviceManager/rest/(?P[^/]+)" + +UNAUTHORIZED = -401 +WRONG_PASSWORD = 1077987870 +OBJECT_NOT_EXIST = 1077948996 +OBJECT_NAME_EXISTS = 1077948993 +OBJECT_ID_NOT_UNIQUE = 1077948997 +LUN_NOT_EXIST = 1077936859 +LUN_ALREADY_IN_GROUP = 1077936862 +HOST_NOT_EXIST = 1077937498 +HOSTGROUP_NOT_EXIST = 1077937500 +HOST_ALREADY_IN_GROUP = 1077937501 +HOST_NOT_IN_GROUP = 1073745412 +HOSTGROUP_NOT_IN_VIEW = 1073804552 +LUNGROUP_NOT_IN_VIEW = 1073804554 +HOSTGROUP_ALREADY_IN_VIEW = 1073804556 +LUNGROUP_ALREADY_IN_VIEW = 1073804560 +MAPPINGVIEW_NOT_EXIST = 1077951819 +PARAMETER_INCORRECT = 50331651 +#: unverified: object still in use (LUN in a group, host with initiators, ...) +IN_USE = 1077949000 +#: unverified: host already in another host group / initiator bound elsewhere +ALREADY_ASSOCIATED = 1077949002 + + +def _fail(code: int, description: str) -> FakeError: + return FakeError(200, {"data": {}, "error": {"code": code, "description": description, + "suggestion": ""}}) + + +def _ok(data: Any = None) -> Reply: + body: dict[str, Any] = {"error": {"code": 0, "description": "0"}} + if data is not None and data != []: + body["data"] = data + return Reply(200, body) + + +def _filter_name(req: Request) -> str | None: + f = req.q("filter") + if not f: + return None + key, _, value = f.partition("::") + if key != "NAME": + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + return value + + +class FakeHuaweiDorado(FakeArray): + vendor = "huawei-dorado" + + def __init__(self, username: str = "admin", password: str = "Admin@storage1", + pools: tuple[str, ...] = ("StoragePool001",)): + super().__init__() + self.username = username + self.password = password + self.device_id = "2102353GTH10L8000008" + self.tokens: set[str] = set() + self.pools = {str(i): {"ID": str(i), "NAME": n, "USAGETYPE": "1"} + for i, n in enumerate(pools)} + self.luns: dict[str, dict[str, Any]] = {} + self.hosts: dict[str, dict[str, Any]] = {} + self.hostgroups: dict[str, dict[str, Any]] = {} # + "hosts": set + self.lungroups: dict[str, dict[str, Any]] = {} # + "luns": set + self.views: dict[str, dict[str, Any]] = {} # + "hostgroups", "lungroups" + self.initiators: dict[str, dict[str, dict[str, Any]]] = { + "iscsi_initiator": {}, "fc_initiator": {}} + self.host_lun_ids: dict[tuple[str, str], int] = {} + #: Answer ``GET /?PARENTID=`` with every initiator, as if + #: the filter were ignored (Huawei's Cinder driver re-checks PARENTID + #: on each row, so clients must not trust the filter). + self.ignore_parent_filter = False + + # -- fake controls ------------------------------------------------------------- + + def expire_sessions(self) -> None: + self.tokens.clear() + + def set_credential(self, **fields: str) -> None: + self.username = fields["username"] + self.password = fields["password"] + + def add_volume(self, name: str, size: int) -> str: + lid = str(self.next_id()) + pool = next(iter(self.pools.values())) + self.luns[lid] = { + "ID": lid, "NAME": name, "TYPE": "11", "PARENTID": pool["ID"], + "PARENTNAME": pool["NAME"], "CAPACITY": str(size // 512), "ALLOCTYPE": "1", + "SECTORSIZE": "512", "HEALTHSTATUS": "1", "RUNNINGSTATUS": "27", + "WWN": "6" + secrets.token_hex(4) + "1001f4e7d" + secrets.token_hex(5) + + f"{int(lid):08x}"[:4], + "DESCRIPTION": "", "ISADD2LUNGROUP": "false"} + return lid + + # -- auth ---------------------------------------------------------------------- + + def _auth(self, req: Request, dev: str) -> None: + if dev != self.device_id or req.headers.get("iBaseToken") not in self.tokens: + raise _fail(UNAUTHORIZED, "The user is not logged in or the session has expired.") + + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return list(self.luns) + + def lio_all_iqns(self) -> list[str]: + return sorted({i["ID"].lower() for i in self.initiators.get("iscsi_initiator", {}).values() if i["PARENTID"]}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + hid = next((h["ID"] for h in self.hosts.values() if host_id in (h["ID"], h["NAME"])), None) + if hid is None: + return [] + return [i["ID"].lower() for i in self.initiators.get("iscsi_initiator", {}).values() + if i["PARENTID"] == hid] + + @route("POST", r"/deviceManager/rest/xx/sessions") + def login(self, req): + body = req.json() or {} + if body.get("username") != self.username or body.get("password") != self.password: + raise _fail(WRONG_PASSWORD, "The user name or password is incorrect.") + if str(body.get("scope")) not in ("0", "1"): + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + token = secrets.token_hex(16) + self.tokens.add(token) + reply = _ok({"deviceid": self.device_id, "iBaseToken": token, "accountstate": 1, + "lastloginip": "10.0.0.1", "level": 2, "pwdchangetime": -1, + "usergroup": "", "userid": body["username"], "username": body["username"], + "userscope": "0"}) + reply.headers["Set-Cookie"] = f"session={secrets.token_hex(8)}; Secure; HttpOnly" + return reply + + @route("DELETE", _D + "/sessions") + def logout(self, req, dev): + self.tokens.discard(req.headers.get("iBaseToken")) + return _ok() + + @route("GET", _D + "/system/") + def system(self, req, dev): + self._auth(req, dev) + return _ok({"ID": self.device_id, "NAME": "fake-dorado", "TYPE": "201", + "PRODUCTMODE": "811", "productModeString": "OceanStor Dorado 5000 V6", + "PRODUCTVERSION": "V600R005C20", "pointRelease": "6.1.5", + "HEALTHSTATUS": "1", "RUNNINGSTATUS": "1"}) + + @route("GET", _D + "/storagepool") + def storagepool(self, req, dev): + self._auth(req, dev) + name = _filter_name(req) + return _ok([p for p in self.pools.values() if name is None or p["NAME"] == name]) + + # -- LUNs -------------------------------------------------------------------------- + + def _lun(self, lid: str) -> dict[str, Any]: + if lid not in self.luns: + raise _fail(LUN_NOT_EXIST, "The LUN does not exist.") + return self.luns[lid] + + @route("GET", _D + "/lun") + def get_luns(self, req, dev): + self._auth(req, dev) + name = _filter_name(req) + return _ok([lun for lun in self.luns.values() if name is None or lun["NAME"] == name]) + + @route("POST", _D + "/lun") + def post_lun(self, req, dev): + self._auth(req, dev) + b = req.json() or {} + name, cap = b.get("NAME"), b.get("CAPACITY") + if not name or len(name) > 31 or not isinstance(cap, int) or cap <= 0: + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + if str(b.get("PARENTID")) not in self.pools: + raise _fail(OBJECT_NOT_EXIST, "The specified storage pool does not exist.") + if any(lun["NAME"] == name for lun in self.luns.values()): + raise _fail(OBJECT_NAME_EXISTS, "The name already exists.") + lid = self.add_volume(name, cap * 512) + self.luns[lid]["PARENTID"] = str(b["PARENTID"]) + self.luns[lid]["PARENTNAME"] = self.pools[str(b["PARENTID"])]["NAME"] + self.luns[lid]["ALLOCTYPE"] = str(b.get("ALLOCTYPE", 1)) + return _ok(self.luns[lid]) + + @route("GET", _D + r"/lun/(?P\d+)") + def get_lun(self, req, dev, lid): + self._auth(req, dev) + return _ok(self._lun(lid)) + + @route("PUT", _D + "/lun/expand") + def expand_lun(self, req, dev): + self._auth(req, dev) + b = req.json() or {} + lun = self._lun(str(b.get("ID"))) + cap = b.get("CAPACITY") + if not isinstance(cap, int) or cap < int(lun["CAPACITY"]): + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + lun["CAPACITY"] = str(cap) + return _ok() + + @route("DELETE", _D + r"/lun/(?P\d+)") + def delete_lun(self, req, dev, lid): + self._auth(req, dev) + self._lun(lid) + if any(lid in g["luns"] for g in self.lungroups.values()): + raise _fail(IN_USE, "The LUN is in a LUN group.") + del self.luns[lid] + return _ok() + + @route("GET", _D + "/lun/associate") + def lun_associate(self, req, dev): + self._auth(req, dev) + if req.q("ASSOCIATEOBJTYPE") != "21": + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + hid = req.q("ASSOCIATEOBJID") or "" + if hid not in self.hosts: + raise _fail(HOST_NOT_EXIST, "The host does not exist.") + rows = [] + for (h, lid), n in sorted(self.host_lun_ids.items()): + if h == hid: + row = dict(self.luns[lid]) + row["ASSOCIATEMETADATA"] = json.dumps({"HostLUNID": n, + "mappingviewId": self._view_of(hid)}) + rows.append(row) + return _ok(rows) + + # -- hosts ---------------------------------------------------------------------------- + + def _host(self, hid: str) -> dict[str, Any]: + if hid not in self.hosts: + raise _fail(OBJECT_NOT_EXIST, "The object does not exist.") + return self.hosts[hid] + + @route("GET", _D + "/host") + def get_hosts(self, req, dev): + self._auth(req, dev) + name = _filter_name(req) + return _ok([h for h in self.hosts.values() if name is None or h["NAME"] == name]) + + @route("POST", _D + "/host") + def post_host(self, req, dev): + self._auth(req, dev) + b = req.json() or {} + name = b.get("NAME") + if not name or len(name) > 31: + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + if any(h["NAME"] == name for h in self.hosts.values()): + raise _fail(OBJECT_NAME_EXISTS, "The name already exists.") + hid = str(self.next_id()) + self.hosts[hid] = {"ID": hid, "NAME": name, "TYPE": "21", + "OPERATIONSYSTEM": str(b.get("OPERATIONSYSTEM", 0)), + "ISADD2HOSTGROUP": "false", "HEALTHSTATUS": "1", + "RUNNINGSTATUS": "1", "DESCRIPTION": b.get("DESCRIPTION", "")} + return _ok(self.hosts[hid]) + + @route("GET", _D + r"/host/(?P\d+)") + def get_host(self, req, dev, hid): + self._auth(req, dev) + return _ok(self._host(hid)) + + @route("DELETE", _D + r"/host/(?P\d+)") + def delete_host(self, req, dev, hid): + self._auth(req, dev) + if hid not in self.hosts: + raise _fail(HOST_NOT_EXIST, "The host does not exist.") + if any(hid in g["hosts"] for g in self.hostgroups.values()): + raise _fail(IN_USE, "The host is in a host group.") + if any(i["PARENTID"] == hid for c in self.initiators.values() for i in c.values()): + raise _fail(IN_USE, "The host has initiators.") + del self.hosts[hid] + return _ok() + + @route("DELETE", _D + "/host/associate") + def host_leave_group(self, req, dev): + self._auth(req, dev) + b = req.json() or {} + g = self.hostgroups.get(str(b.get("ID"))) + if g is None: + raise _fail(HOSTGROUP_NOT_EXIST, "The host group does not exist.") + hid = str(b.get("ASSOCIATEOBJID")) + if hid not in g["hosts"]: + raise _fail(HOST_NOT_IN_GROUP, "The host is not in the host group.") + g["hosts"].discard(hid) + self.hosts[hid]["ISADD2HOSTGROUP"] = "false" + self._refresh() + return _ok() + + # -- initiators ------------------------------------------------------------------------- + + def _ini_lookup(self, coll: str, iid: str) -> dict[str, Any] | None: + for k, v in self.initiators[coll].items(): + if k.lower() == iid.lower(): + return v + return None + + @route("GET", _D + r"/(?Piscsi_initiator|fc_initiator)") + def list_initiators(self, req, dev, coll): + self._auth(req, dev) + parent = None if self.ignore_parent_filter else req.q("PARENTID") + return _ok([i for i in self.initiators[coll].values() + if parent is None or i["PARENTID"] == parent]) + + @route("POST", _D + r"/(?Piscsi_initiator|fc_initiator)") + def add_initiator(self, req, dev, coll): + self._auth(req, dev) + b = req.json() or {} + iid = str(b.get("ID") or "") + if coll == "fc_initiator" and (len(iid) != 16 or any( + c not in "0123456789abcdef" for c in iid)): + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + if not iid: + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + if self._ini_lookup(coll, iid) is not None: + raise _fail(OBJECT_ID_NOT_UNIQUE, "The ID already exists.") + self.initiators[coll][iid] = { + "ID": iid, "TYPE": "222" if coll == "iscsi_initiator" else "223", + "ISFREE": "true", "PARENTID": "", "PARENTNAME": "", "PARENTTYPE": "", + "RUNNINGSTATUS": "28", "HEALTHSTATUS": "1", "MULTIPATHTYPE": "0"} + return _ok(self.initiators[coll][iid]) + + @route("GET", _D + r"/(?Piscsi_initiator|fc_initiator)/(?P[^/]+)") + def get_initiator(self, req, dev, coll, iid): + self._auth(req, dev) + ini = self._ini_lookup(coll, iid) + if ini is None: + raise _fail(OBJECT_NOT_EXIST, "The object does not exist.") + return _ok(ini) + + @route("PUT", _D + r"/(?Piscsi_initiator|fc_initiator)/(?P[^/]+)") + def put_initiator(self, req, dev, coll, iid): + self._auth(req, dev) + b = req.json() or {} + if iid in ("remove_iscsi_from_host", "remove_fc_from_host"): + ini = self._ini_lookup(coll, str(b.get("ID") or "")) + if ini is None: + raise _fail(OBJECT_NOT_EXIST, "The object does not exist.") + ini.update(ISFREE="true", PARENTID="", PARENTNAME="", PARENTTYPE="") + return _ok() + ini = self._ini_lookup(coll, iid) + if ini is None: + raise _fail(OBJECT_NOT_EXIST, "The object does not exist.") + if "PARENTID" in b: + hid = str(b["PARENTID"]) + host = self._host(hid) + if str(b.get("PARENTTYPE")) != "21": + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + if ini["ISFREE"] == "false" and ini["PARENTID"] != hid: + raise _fail(ALREADY_ASSOCIATED, "The initiator has been added to another host.") + ini.update(ISFREE="false", PARENTID=hid, PARENTNAME=host["NAME"], PARENTTYPE="21") + return _ok() + + # -- groups and views --------------------------------------------------------------------- + + def _create_named(self, store: dict[str, dict[str, Any]], req: Request, + obj_type: str, **sets: set) -> Reply: + b = req.json() or {} + name = b.get("NAME") + if not name or len(name) > 31: + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + if any(o["NAME"] == name for o in store.values()): + raise _fail(OBJECT_NAME_EXISTS, "The name already exists.") + oid = str(self.next_id()) + store[oid] = {"ID": oid, "NAME": name, "TYPE": obj_type, "DESCRIPTION": "", **sets} + return _ok(self._public(store[oid])) + + @staticmethod + def _public(obj: dict[str, Any]) -> dict[str, Any]: + return {k: v for k, v in obj.items() if not isinstance(v, set)} + + def _list_named(self, store, req) -> Reply: + name = _filter_name(req) + return _ok([self._public(o) for o in store.values() if name is None or o["NAME"] == name]) + + @route("GET", _D + "/hostgroup") + def get_hostgroups(self, req, dev): + self._auth(req, dev) + return self._list_named(self.hostgroups, req) + + @route("POST", _D + "/hostgroup") + def post_hostgroup(self, req, dev): + self._auth(req, dev) + return self._create_named(self.hostgroups, req, "14", hosts=set()) + + @route("DELETE", _D + r"/hostgroup/(?P\d+)") + def delete_hostgroup(self, req, dev, gid): + self._auth(req, dev) + g = self.hostgroups.get(gid) + if g is None: + raise _fail(HOSTGROUP_NOT_EXIST, "The host group does not exist.") + if g["hosts"] or any(gid in v["hostgroups"] for v in self.views.values()): + raise _fail(IN_USE, "The host group is in use.") + del self.hostgroups[gid] + return _ok() + + @route("POST", _D + "/hostgroup/associate") + def hostgroup_add_host(self, req, dev): + self._auth(req, dev) + b = req.json() or {} + g = self.hostgroups.get(str(b.get("ID"))) + if g is None: + raise _fail(HOSTGROUP_NOT_EXIST, "The host group does not exist.") + hid = str(b.get("ASSOCIATEOBJID")) + if hid not in self.hosts: + raise _fail(HOST_NOT_EXIST, "The host does not exist.") + if hid in g["hosts"]: + raise _fail(HOST_ALREADY_IN_GROUP, "The host already exists in the host group.") + if any(hid in o["hosts"] for o in self.hostgroups.values()): + raise _fail(ALREADY_ASSOCIATED, "The host already belongs to another host group.") + g["hosts"].add(hid) + self.hosts[hid]["ISADD2HOSTGROUP"] = "true" + self._refresh() + return _ok() + + @route("GET", _D + "/hostgroup/associate") + def hostgroups_of(self, req, dev): + self._auth(req, dev) + t, oid = req.q("ASSOCIATEOBJTYPE"), req.q("ASSOCIATEOBJID") or "" + if t == "21": + return _ok([self._public(g) for g in self.hostgroups.values() if oid in g["hosts"]]) + if t == "245": + v = self.views.get(oid) + return _ok([self._public(self.hostgroups[g]) for g in v["hostgroups"]] if v else []) + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + + @route("GET", _D + "/lungroup") + def get_lungroups(self, req, dev): + self._auth(req, dev) + return self._list_named(self.lungroups, req) + + @route("POST", _D + "/lungroup") + def post_lungroup(self, req, dev): + self._auth(req, dev) + return self._create_named(self.lungroups, req, "256", luns=set()) + + @route("DELETE", _D + r"/lungroup/(?P\d+)") + def delete_lungroup(self, req, dev, gid): + self._auth(req, dev) + g = self.lungroups.get(gid) + if g is None: + raise _fail(OBJECT_NOT_EXIST, "The object does not exist.") + if g["luns"] or any(gid in v["lungroups"] for v in self.views.values()): + raise _fail(IN_USE, "The LUN group is in use.") + del self.lungroups[gid] + return _ok() + + @route("POST", _D + "/lungroup/associate") + def lungroup_add_lun(self, req, dev): + self._auth(req, dev) + b = req.json() or {} + g = self.lungroups.get(str(b.get("ID"))) + if g is None: + raise _fail(OBJECT_NOT_EXIST, "The object does not exist.") + if str(b.get("ASSOCIATEOBJTYPE")) != "11": + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + lid = str(b.get("ASSOCIATEOBJID")) + self._lun(lid) + if lid in g["luns"]: + raise _fail(LUN_ALREADY_IN_GROUP, "The LUN is already in the LUN group.") + g["luns"].add(lid) + self.luns[lid]["ISADD2LUNGROUP"] = "true" + self._refresh() + return _ok() + + @route("DELETE", _D + "/lungroup/associate") + def lungroup_remove_lun(self, req, dev): + self._auth(req, dev) + b = req.json() or {} + g = self.lungroups.get(str(b.get("ID"))) + lid = str(b.get("ASSOCIATEOBJID")) + if g is None or lid not in g["luns"]: + raise _fail(OBJECT_NOT_EXIST, "The object does not exist.") + g["luns"].discard(lid) + if lid in self.luns and not any(lid in o["luns"] for o in self.lungroups.values()): + self.luns[lid]["ISADD2LUNGROUP"] = "false" + self._refresh() + return _ok() + + @route("GET", _D + "/lungroup/associate") + def lungroups_of(self, req, dev): + self._auth(req, dev) + if req.q("ASSOCIATEOBJTYPE") != "11": + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + lid = req.q("ASSOCIATEOBJID") or "" + self._lun(lid) + return _ok([self._public(g) for g in self.lungroups.values() if lid in g["luns"]]) + + @route("GET", _D + "/mappingview") + def get_views(self, req, dev): + self._auth(req, dev) + return self._list_named(self.views, req) + + @route("POST", _D + "/mappingview") + def post_view(self, req, dev): + self._auth(req, dev) + return self._create_named(self.views, req, "245", hostgroups=set(), lungroups=set()) + + @route("DELETE", _D + r"/mappingview/(?P\d+)") + def delete_view(self, req, dev, vid): + self._auth(req, dev) + v = self.views.get(vid) + if v is None: + raise _fail(MAPPINGVIEW_NOT_EXIST, "The mapping view does not exist.") + if v["hostgroups"] or v["lungroups"]: + raise _fail(IN_USE, "The mapping view is in use.") + del self.views[vid] + return _ok() + + @route("PUT", _D + r"/mappingview/(?Pcreate_associate|remove_associate)") + def view_associate(self, req, dev, action): + self._auth(req, dev) + b = req.json() or {} + v = self.views.get(str(b.get("ID"))) + if v is None: + raise _fail(MAPPINGVIEW_NOT_EXIST, "The mapping view does not exist.") + t, oid = str(b.get("ASSOCIATEOBJTYPE")), str(b.get("ASSOCIATEOBJID")) + if t == "14": + key, store, already, missing = ("hostgroups", self.hostgroups, + HOSTGROUP_ALREADY_IN_VIEW, HOSTGROUP_NOT_IN_VIEW) + elif t == "256": + key, store, already, missing = ("lungroups", self.lungroups, + LUNGROUP_ALREADY_IN_VIEW, LUNGROUP_NOT_IN_VIEW) + else: + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + if oid not in store: + raise _fail(OBJECT_NOT_EXIST, "The object does not exist.") + if action == "create_associate": + if oid in v[key]: + raise _fail(already, "The group is already in the mapping view.") + if v[key]: + raise _fail(PARAMETER_INCORRECT, "A mapping view holds one group of each kind.") + v[key].add(oid) + else: + if oid not in v[key]: + raise _fail(missing, "The group is not in the mapping view.") + v[key].discard(oid) + self._refresh() + return _ok() + + # -- visibility ------------------------------------------------------------------------------- + + def _view_of(self, hid: str) -> str: + for v in self.views.values(): + if any(hid in self.hostgroups[g]["hosts"] for g in v["hostgroups"]): + return v["ID"] + return "" + + def _refresh(self) -> None: + """Recompute which LUNs each host sees; keep existing host LUN ids.""" + visible: set[tuple[str, str]] = set() + for v in self.views.values(): + hosts = {h for g in v["hostgroups"] for h in self.hostgroups[g]["hosts"]} + luns = {lun for g in v["lungroups"] for lun in self.lungroups[g]["luns"]} + visible |= {(h, lun) for h in hosts for lun in luns} + self.host_lun_ids = {k: n for k, n in self.host_lun_ids.items() if k in visible} + for h, lun in sorted(visible - set(self.host_lun_ids)): + used = {n for (hh, _), n in self.host_lun_ids.items() if hh == h} + self.host_lun_ids[(h, lun)] = next(n for n in range(1, 4096) if n not in used) + + +def contract_setup() -> ContractSetup: + fake = FakeHuaweiDorado(username="admin", password="Admin@storage1") + return ContractSetup( + fake=fake, + config=make_config("huawei-dorado", "https://dorado.fake.example:8088", + storage_pool="StoragePool001"), + credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("Admin@storage1")}), + bad_credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_volume(name, size), + ) diff --git a/simplyblock_core/storage_arrays/testing/ibm_flashsystem.py b/simplyblock_core/storage_arrays/testing/ibm_flashsystem.py new file mode 100644 index 0000000000..3446b2f278 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/ibm_flashsystem.py @@ -0,0 +1,418 @@ +"""Fake IBM Storage Virtualize REST API (the subset the adapter uses). + +Mirrors the real API's shape: every command is ``POST /rest[/v1]/[/]`` +with CLI flags as JSON; failures are HTTP 500 with the CLI's ``CMMVCnnnnE`` +message as plain text; an unknown/expired token is HTTP 403. Quirks kept: + +* values are strings (ids, capacities, SCSI ids); +* ``capacity`` is human-readable (``10.00GB``) unless ``bytes`` is passed; +* detail views (``lsvdisk ``) are a list: the volume, then its copies; +* a one-row ``lsvdiskhostmap`` comes back as a bare dict; +* object ids are the lowest free integer, so a deleted volume's id is reused; +* ``mkvdiskhostmap`` to a second host needs ``force`` (CMMVC6071E); +* FC WWPNs that are not logged in need ``force`` on mkhost/addhostport; +* ``mkhost`` takes only one of ``-iscsiname`` / ``-fcwwpn`` / ``-nqn``; +* a port owned by another host fails with CMMVC6578E (iSCSI), CMMVC5867E + (FC) or CMMVC9328E (NVMe). +""" +from __future__ import annotations + +import secrets +from typing import Any + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +_ROOT = r"/rest(?:/v1)?" + +E_NOT_EXIST = "CMMVC5753E The specified object does not exist or is not a suitable candidate." +E_NAME_RULES = ("CMMVC5754E The specified object does not exist, or the name supplied " + "does not meet the naming rules.") +E_IN_REQUEST = "CMMVC5842E The action failed because an object that was in the request does not exist." +E_EXISTS = "CMMVC6035E The action failed as the object already exists." +# Codes as handled by IBM's CSI driver (array_mediator_svc.py) and Cinder +# (storwize_const.py); the message texts are approximations. +E_IQN = ("CMMVC6578E The command has failed because the iSCSI name is already assigned " + "or is not valid.") +E_NQN = "CMMVC9328E The command failed because the NQN is already assigned to a host." +E_WWPN = ("CMMVC5867E The action failed because the worldwide port name is already " + "assigned or is not valid.") +E_ALREADY_MAPPED = ("CMMVC5878E The virtual disk (VDisk)-to-host mapping was not created " + "because this VDisk is already mapped to this host.") +E_MULTI_HOST = ("CMMVC6071E The VDisk-to-host mapping was not created because the VDisk is " + "already mapped to a host.") +E_VDISK_MAPPED = ("CMMVC5840E The virtual disk (VDisk) was not deleted because it is mapped " + "to a host or because it is part of a FlashCopy or Remote Copy mapping, " + "or is involved in an image mode migrate.") +E_HOST_MAPPED = ("CMMVC5871E The action failed because one or more of the configured port " + "names is in a mapping.") +E_BAD_NAME = ("CMMVC6527E The name that you have entered is not valid. The name can contain " + "letters, numbers, spaces, periods, dashes, and underscores. The name must " + "begin with a letter or an underscore.") +E_PARAM = "CMMVC5707E Required parameters are missing." +E_EXCLUSIVE = "CMMVC5731E Parameter -fcwwpn is incompatible with parameter -iscsiname." +E_SHRINK = "CMMVC5709E [-size] is not a supported parameter." + + +def _cli_error(text: str) -> FakeError: + return FakeError(500, text.encode(), {"Content-Type": "text/plain"}) + + +def _human(size: int) -> str: + for unit, div in (("TB", 1 << 40), ("GB", 1 << 30), ("MB", 1 << 20)): + if size >= div: + return f"{size / div:.2f}{unit}" + return f"{size}B" + + +def _units(unit: str | None) -> int: + return 1 << (10 * ["b", "kb", "mb", "gb", "tb", "pb"].index((unit or "mb").lower())) + + +def _valid_name(name: str) -> bool: + return bool(name) and len(name) <= 63 and (name[0].isalpha() or name[0] == "_") and all( + c.isalnum() or c in "._- " for c in name) + + +class FakeIbmFlashSystem(FakeArray): + vendor = "ibm-flashsystem" + + def __init__(self, username: str = "superuser", password: str = "passw0rd", + pools: tuple[str, ...] = ("Pool0",)): + super().__init__() + self.username = username + self.password = password + self.pools = {name: str(i) for i, name in enumerate(pools)} + self.tokens: set[str] = set() + self.vdisks: dict[str, dict[str, Any]] = {} # id -> vdisk + self.hosts: dict[str, dict[str, Any]] = {} # name -> host + self.maps: list[dict[str, Any]] = [] # {vdisk_id, host, scsi} + self.commands: list[tuple[str, str | None, dict[str, Any]]] = [] + + # -- fake controls ------------------------------------------------------------ + + def expire_sessions(self) -> None: + self.tokens.clear() + + def set_credential(self, **fields: str) -> None: + self.username = fields["username"] + self.password = fields["password"] + + @staticmethod + def _free_id(used) -> str: + n = 0 + while str(n) in used: + n += 1 + return str(n) + + def add_volume(self, name: str, size: int, thin: bool = False) -> str: + vid = self._free_id(self.vdisks) + self.vdisks[vid] = {"id": vid, "name": name, "capacity": size, + "vdisk_UID": "600507680C8" + secrets.token_hex(11).upper()[:21], + "mdisk_grp_name": next(iter(self.pools)), "IO_group_name": "io_grp0", + "thin": thin} + return vid + + # -- dispatch ------------------------------------------------------------------- + + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return list(self.vdisks) + + def lio_all_iqns(self) -> list[str]: + return sorted({str(i).lower() for h in self.hosts.values() for i in h.get("iscsiname", [])}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + for name, h in self.hosts.items(): + if host_id in (name, str(h.get("id", ""))): + return [str(i).lower() for i in h.get("iscsiname", [])] + return [] + + @route("POST", _ROOT + "/auth") + def auth(self, req: Request): + if (req.headers.get("X-Auth-Username") != self.username + or req.headers.get("X-Auth-Password") != self.password): + raise FakeError(403, {"code": 403, "message": "Forbidden"}) + token = secrets.token_hex(16) + self.tokens.add(token) + return Reply(200, {"token": token}) + + @route("POST", _ROOT + r"/(?P[a-z]+)(?:/(?P[^/]+))?") + def command(self, req: Request, cmd: str, obj: str | None = None): + if cmd == "auth": # pragma: no cover - matched above + return self.auth(req) + if req.headers.get("X-Auth-Token") not in self.tokens: + raise FakeError(403, {"code": 403, "message": "Forbidden"}) + opts = req.json() or {} + self.commands.append((cmd, obj, opts)) + handler = getattr(self, f"_c_{cmd}", None) + if handler is None: + raise FakeError(404, "Not Found") + result = handler(obj, opts) + return Reply(200, result) + + # -- lookups ---------------------------------------------------------------------- + + def _vdisk(self, ref: str | None) -> dict[str, Any]: + if ref in self.vdisks: + return self.vdisks[ref] + for v in self.vdisks.values(): + if v["name"] == ref: + return v + raise _cli_error(E_NOT_EXIST) + + def _host(self, ref: str | None) -> dict[str, Any]: + if ref in self.hosts: + return self.hosts[ref] + for h in self.hosts.values(): + if h["id"] == ref: + return h + raise _cli_error(E_NOT_EXIST) + + def _vdisk_row(self, v: dict[str, Any], opts: dict[str, Any]) -> dict[str, Any]: + cap = str(v["capacity"]) if opts.get("bytes") else _human(v["capacity"]) + return {"id": v["id"], "name": v["name"], "IO_group_id": "0", + "IO_group_name": v["IO_group_name"], "status": "online", + "mdisk_grp_id": self.pools[v["mdisk_grp_name"]], + "mdisk_grp_name": v["mdisk_grp_name"], "capacity": cap, "type": "striped", + "FC_id": "", "FC_name": "", "RC_id": "", "RC_name": "", + "vdisk_UID": v["vdisk_UID"], "fc_map_count": "0", "copy_count": "1", + "fast_write_state": "empty", "se_copy_count": "1" if v["thin"] else "0", + "RC_change": "no", "compressed_copy_count": "0", "parent_mdisk_grp_id": "0", + "parent_mdisk_grp_name": v["mdisk_grp_name"], "formatting": "no", + "encrypt": "no", "volume_id": v["id"], "volume_name": v["name"], + "function": ""} + + # -- system ------------------------------------------------------------------------- + + def _c_lssystem(self, obj, opts): + return {"id": "000002042140A0B2", "name": "fake-fs7300", "location": "local", + "code_level": "8.6.0.4 (build 169.12.2403211122000)", + "product_name": "IBM FlashSystem 7300"} + + # -- volumes -------------------------------------------------------------------------- + + def _c_lsvdisk(self, obj, opts): + if obj is not None: + v = self._vdisk(obj) + row = self._vdisk_row(v, opts) + cap = row["capacity"] + copy = {"copy_id": "0", "status": "online", "sync": "yes", "primary": "yes", + "mdisk_grp_name": v["mdisk_grp_name"], "type": "striped", + "capacity": cap, "real_capacity": cap if not v["thin"] else "0.00MB", + "autoexpand": "on" if v["thin"] else "", "se_copy": "yes" if v["thin"] else "no"} + return [row, copy] + rows = [self._vdisk_row(v, opts) for v in self.vdisks.values()] + fv = opts.get("filtervalue") + if fv: + key, _, value = str(fv).partition("=") + rows = [r for r in rows if str(r.get(key)) == value] + return rows + + def _c_mkvolume(self, obj, opts): + name, pool = opts.get("name"), opts.get("pool") + if "size" not in opts or not pool: + raise _cli_error(E_PARAM) + if pool not in self.pools and pool not in self.pools.values(): + raise _cli_error(E_NAME_RULES) + if name is not None and not _valid_name(name): + raise _cli_error(E_BAD_NAME) + if any(v["name"] == name for v in self.vdisks.values()): + raise _cli_error(E_EXISTS) + size = int(opts["size"]) * _units(opts.get("unit")) + size = -(-size // 512) * 512 + vid = self.add_volume(name or "", size, thin=bool(opts.get("thin"))) + v = self.vdisks[vid] + if not name: + v["name"] = f"volume{vid}" + if pool in self.pools: + v["mdisk_grp_name"] = pool + if opts.get("iogrp") is not None: + v["IO_group_name"] = str(opts["iogrp"]) + return {"id": vid, "message": f"Volume, id [{vid}], successfully created"} + + def _c_expandvdisksize(self, obj, opts): + v = self._vdisk(obj) + if "size" not in opts: + raise _cli_error(E_PARAM) + delta = int(opts["size"]) * _units(opts.get("unit")) + if delta <= 0: + raise _cli_error(E_SHRINK) + v["capacity"] += -(-delta // 512) * 512 + + def _c_rmvolume(self, obj, opts): + v = self._vdisk(obj) + if any(m["vdisk_id"] == v["id"] for m in self.maps) and not opts.get("removehostmappings"): + raise _cli_error(E_VDISK_MAPPED) + self.maps = [m for m in self.maps if m["vdisk_id"] != v["id"]] + del self.vdisks[v["id"]] + + # -- hosts ------------------------------------------------------------------------------ + + def _owner(self, kind: str, value: str) -> str | None: + for h in self.hosts.values(): + if value.lower() in (p.lower() for p in h[kind]): + return h["name"] + return None + + def _parse_ports(self, opts: dict[str, Any]) -> dict[str, list[str]]: + ports: dict[str, list[str]] = {"iscsiname": [], "fcwwpn": [], "nqn": []} + if opts.get("iscsiname"): + ports["iscsiname"] = [p for p in str(opts["iscsiname"]).split(",") if p] + if opts.get("fcwwpn"): + wwpns = [p for p in str(opts["fcwwpn"]).split(":") if p] + for w in wwpns: + if len(w) != 16 or any(c not in "0123456789abcdefABCDEF" for c in w): + raise _cli_error(E_WWPN) + if not opts.get("force"): + # The fake has no fabric: no WWPN is ever logged in. + raise _cli_error(E_WWPN) + ports["fcwwpn"] = [w.upper() for w in wwpns] + if opts.get("nqn"): + ports["nqn"] = [p for p in str(opts["nqn"]).split(",") if p] + return ports + + def _claim(self, host: str | None, ports: dict[str, list[str]]) -> None: + errors = {"iscsiname": E_IQN, "fcwwpn": E_WWPN, "nqn": E_NQN} + for kind, values in ports.items(): + for value in values: + if self._owner(kind, value) is not None: # even when it is this host + raise _cli_error(errors[kind]) + + def _c_mkhost(self, obj, opts): + name = opts.get("name") + if not name: + raise _cli_error(E_PARAM) + if not _valid_name(name): + raise _cli_error(E_BAD_NAME) + if name in self.hosts: + raise _cli_error(E_EXISTS) + protocol = opts.get("protocol", "scsi") + if protocol not in ("scsi", "iscsi", "fcscsi", "tcpnvme", "rdmanvme", "fcnvme", "nvme"): + raise _cli_error(E_PARAM) + if sum(1 for k in ("iscsiname", "fcwwpn", "nqn") if opts.get(k)) > 1: + raise _cli_error(E_EXCLUSIVE) + ports = self._parse_ports(opts) + if not any(ports.values()): + raise _cli_error(E_PARAM) + if ports["nqn"] and protocol not in ("tcpnvme", "rdmanvme", "fcnvme", "nvme"): + raise _cli_error(E_PARAM) + self._claim(None, ports) + hid = self._free_id({h["id"] for h in self.hosts.values()}) + self.hosts[name] = {"id": hid, "name": name, "protocol": protocol, + "iscsiname": ports["iscsiname"], "fcwwpn": ports["fcwwpn"], + "nqn": ports["nqn"]} + return {"id": hid, "message": f"Host, id [{hid}], successfully created"} + + def _c_addhostport(self, obj, opts): + h = self._host(obj) + ports = self._parse_ports(opts) + if not any(ports.values()): + raise _cli_error(E_PARAM) + self._claim(h["name"], ports) + for kind, values in ports.items(): + h[kind].extend(values) + + def _c_lshost(self, obj, opts): + if obj is None: + return [{"id": h["id"], "name": h["name"], "port_count": str( + len(h["iscsiname"]) + len(h["fcwwpn"]) + len(h["nqn"])), + "iogrp_count": "4", "status": "online", "protocol": h["protocol"]} + for h in self.hosts.values()] + h = self._host(obj) + nodes = ([{"WWPN": w, "node_logged_in_count": "0", "state": "offline"} + for w in h["fcwwpn"]] + + [{"iscsi_name": i, "node_logged_in_count": "0", "state": "offline"} + for i in h["iscsiname"]] + + [{"nqn": n, "node_logged_in_count": "0", "state": "offline"} + for n in h["nqn"]]) + return {"id": h["id"], "name": h["name"], "port_count": str(len(nodes)), + "type": "generic", "iogrp_count": "4", "status": "offline", + "protocol": h["protocol"], "host_cluster_id": "", "host_cluster_name": "", + "mapping_count": str(sum(1 for m in self.maps if m["host"] == h["name"])), + "nodes": nodes} + + def _c_rmhost(self, obj, opts): + h = self._host(obj) + if any(m["host"] == h["name"] for m in self.maps) and not opts.get("force"): + raise _cli_error(E_HOST_MAPPED) + self.maps = [m for m in self.maps if m["host"] != h["name"]] + del self.hosts[h["name"]] + + # -- mappings ------------------------------------------------------------------------------- + + def _map_row(self, m: dict[str, Any]) -> dict[str, Any]: + v, h = self.vdisks[m["vdisk_id"]], self.hosts[m["host"]] + return {"id": v["id"], "name": v["name"], "SCSI_id": str(m["scsi"]), + "host_id": h["id"], "host_name": h["name"], "vdisk_UID": v["vdisk_UID"], + "IO_group_id": "0", "IO_group_name": v["IO_group_name"], + "mapping_type": "private", "host_cluster_id": "", "host_cluster_name": "", + "protocol": h["protocol"]} + + def _c_lsvdiskhostmap(self, obj, opts): + v = self._vdisk(obj) + rows = [self._map_row(m) for m in self.maps if m["vdisk_id"] == v["id"]] + return rows[0] if len(rows) == 1 else rows # quirk: one row -> bare dict + + def _c_mkvdiskhostmap(self, obj, opts): + v = self._vdisk(obj) + if not opts.get("host"): + raise _cli_error(E_PARAM) + h = self._host(opts["host"]) + mine = [m for m in self.maps if m["vdisk_id"] == v["id"]] + if any(m["host"] == h["name"] for m in mine): + raise _cli_error(E_ALREADY_MAPPED) + if mine and not opts.get("force"): + raise _cli_error(E_MULTI_HOST) + used = {m["scsi"] for m in self.maps if m["host"] == h["name"]} + scsi = int(opts["scsi"]) if opts.get("scsi") is not None else next( + n for n in range(4096) if n not in used) + self.maps.append({"vdisk_id": v["id"], "host": h["name"], "scsi": scsi}) + return {"id": str(scsi), + "message": f"Virtual Disk to Host map, id [{scsi}], successfully created"} + + def _c_rmvdiskhostmap(self, obj, opts): + v = self._vdisk(obj) + if not opts.get("host"): + raise _cli_error(E_PARAM) + h = self._host(opts["host"]) + before = len(self.maps) + self.maps = [m for m in self.maps + if not (m["vdisk_id"] == v["id"] and m["host"] == h["name"])] + if len(self.maps) == before: + raise _cli_error(E_IN_REQUEST) + + +def contract_setup() -> ContractSetup: + fake = FakeIbmFlashSystem(username="superuser", password="passw0rd") + return ContractSetup( + fake=fake, + config=make_config("ibm-flashsystem", "https://flashsystem.fake.example:7443", + pool="Pool0"), + credential=Credential({"username": SecretStr("superuser"), + "password": SecretStr("passw0rd")}), + bad_credential=Credential({"username": SecretStr("superuser"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + Protocol.NVME_TCP: ( + (Initiator(protocol=Protocol.NVME_TCP, + address="nqn.2014-08.org.nvmexpress:uuid:host-a"),), + (Initiator(protocol=Protocol.NVME_TCP, + address="nqn.2014-08.org.nvmexpress:uuid:host-b"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_volume(name, size), + ) diff --git a/simplyblock_core/storage_arrays/testing/infinidat.py b/simplyblock_core/storage_arrays/testing/infinidat.py new file mode 100644 index 0000000000..d319c90ec5 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/infinidat.py @@ -0,0 +1,343 @@ +"""Fake Infinidat InfiniBox REST API (``/api/rest``), the subset the adapter +uses. + +Quirks reproduced: every reply is a ``{"result", "error", "metadata"}`` +envelope (metadata ``page``/``page_size``/``pages_total``/ +``number_of_objects``); lists are paged and filtered with +``field=eq:value``; destructive requests without ``?approved=true`` get +``APPROVAL_REQUIRED``; object ids are integers; volume ``serial`` is the +NAA-6 identifier without its leading ``6``; port types are +case-insensitive and FC addresses are accepted with or without colons; +``GET /system`` reports ``model``/``full_model``/integer ``serial_number``. + +Codes taken from Infinidat's infinisdk / CSI driver: ``VOLUME_NOT_FOUND``, +``HOST_NOT_FOUND``, ``POOL_NOT_FOUND``, ``LUN_NOT_FOUND``, +``MAPPING_ALREADY_EXISTS``, ``APPROVAL_REQUIRED``. The name-conflict, port +ownership, mapped-volume and host-with-LUNs codes, the 403 status of +``APPROVAL_REQUIRED`` and LUN numbering from 1 are the fake's own reading. +Infinidat's simulator (``infinisim``) is not publicly available. +""" +from __future__ import annotations + +from typing import Any + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol, normalize_wwpn +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +_R = "/api/rest" +_ID = r"(?P\d+)" +_SYSTEM_ID = "000004e2" + + +def _err(status: int, code: str, message: str) -> FakeError: + return FakeError(status, { + "result": None, + "error": {"code": code, "message": message, "is_remote": False, "data": None, + "reasons": [], "severity": "ERROR"}, + "metadata": {"ready": True}}) + + +def _ok(result: Any, status: int = 200, metadata: dict[str, Any] | None = None) -> Reply: + return Reply(status, {"result": result, "error": None, + "metadata": metadata or {"ready": True}}) + + +def _port_address(ptype: str, address: str) -> str: + if ptype == "FC": + try: + return normalize_wwpn(address) + except ValueError: + raise _err(400, "INVALID_PORT_ADDRESS", f"Invalid FC address {address}") from None + return address.lower() + + +class FakeInfiniBox(FakeArray): + vendor = "infinidat-infinibox" + + def __init__(self, username: str = "admin", password: str = "ibox-pass"): + super().__init__() + self.username = username + self.password = password + self.pools: dict[int, dict[str, Any]] = {} + self.volumes: dict[int, dict[str, Any]] = {} + self.hosts: dict[int, dict[str, Any]] = {} + self.luns: list[dict[str, Any]] = [] # {host_id, volume_id, lun, id} + #: require ``approved=true`` on every non-GET request, not only on + #: destructive ones (which requests need approval varies by operation + #: and version; infinisdk approves all of them) + self.approval_for_all = False + self.add_pool("sb-pool") + + # -- fake controls ------------------------------------------------------ + + def set_credential(self, **fields: str) -> None: + self.username = fields.get("username", self.username) + self.password = fields["password"] + + def add_pool(self, name: str) -> int: + pid = self.next_id() + self.pools[pid] = {"id": pid, "name": name, "physical_capacity": 1 << 50, + "virtual_capacity": 1 << 52, "state": "NORMAL"} + return pid + + def add_volume(self, name: str, size: int, pool_id: int | None = None) -> int: + vid = self.next_id() + self.volumes[vid] = { + "id": vid, "name": name, "size": size, "used": 0, "allocated": 0, + "pool_id": pool_id or next(iter(self.pools)), "provtype": "THIN", + "type": "MASTER", "mapped": False, "write_protected": False, + "serial": f"742b0f{_SYSTEM_ID}{vid:017x}", "created_at": 1763600000000, + "parent_id": 0, "family_id": vid, "rmr_source": False, "rmr_target": False, + } + return vid + + # -- dispatch helpers --------------------------------------------------------- + + def _auth(self, req: Request) -> None: + if req.basic_auth() != (self.username, self.password): + raise _err(401, "UNAUTHORIZED", "Invalid credentials") + if self.approval_for_all and req.method != "GET": + self._approved(req, f"{req.method} {req.path}") + + @staticmethod + def _approved(req: Request, what: str) -> None: + if (req.q("approved") or "").lower() != "true": + raise _err(403, "APPROVAL_REQUIRED", f"{what} requires approval") + + def _page(self, req: Request, items: list[dict[str, Any]]) -> Reply: + for key, values in req.query.items(): + if key in ("page", "page_size", "approved", "fields", "sort"): + continue + raw = values[0] + op, _, value = raw.partition(":") if ":" in raw else ("eq", "", raw) + if op != "eq": + raise _err(400, "UNSUPPORTED_FILTER", f"fake: filter op {op}") + items = [i for i in items if str(i.get(key)) == value] + page = int(req.q("page") or 1) + size = int(req.q("page_size") or 50) + if not 1 <= size <= 1000: + raise _err(400, "BAD_PAGE_SIZE", "page_size must be 1..1000") + total = len(items) + pages = max(1, -(-total // size)) + chunk = items[(page - 1) * size: page * size] + return _ok(chunk, metadata={"ready": True, "number_of_objects": total, + "page_size": size, "pages_total": pages, "page": page}) + + def _vol(self, vid: int) -> dict[str, Any]: + if vid not in self.volumes: + raise _err(404, "VOLUME_NOT_FOUND", f"Volume {vid} not found") + return self.volumes[vid] + + def _host(self, hid: int) -> dict[str, Any]: + if hid not in self.hosts: + raise _err(404, "HOST_NOT_FOUND", f"Host {hid} not found") + return self.hosts[hid] + + def _host_view(self, h: dict[str, Any]) -> dict[str, Any]: + luns = [self._lun_view(m) for m in self.luns if m["host_id"] == h["id"]] + return {**h, "ports": [dict(p) for p in h["ports"]], "luns": luns} + + def _lun_view(self, m: dict[str, Any]) -> dict[str, Any]: + return {"id": m["id"], "lun": m["lun"], "volume_id": m["volume_id"], + "host_id": m["host_id"], "host_cluster_id": 0, "clustered": False} + + # -- system / pools -------------------------------------------------------------- + + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return [str(i) for i in self.volumes] + + def lio_all_iqns(self) -> list[str]: + return sorted({p["address"].lower() for h in self.hosts.values() for p in h["ports"] if p["type"] == "ISCSI"}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + for h in self.hosts.values(): + if host_id in (str(h["id"]), h["name"]): + return [p["address"].lower() for p in h["ports"] if p["type"] == "ISCSI"] + return [] + + @route("GET", _R + "/system") + def system(self, req): + self._auth(req) + return _ok({"name": "ibox-fake", "serial_number": 1250, "model": "F4304", + "full_model": "InfiniBox F4304", "version": "7.3.10.0", + "product_id": "INFINIBOX", "wwnn": "57:42:b0:f0:00:04:e2:00"}) + + @route("GET", _R + "/pools") + def list_pools(self, req): + self._auth(req) + return self._page(req, list(self.pools.values())) + + # -- volumes -------------------------------------------------------------------- + + @route("GET", _R + "/volumes") + def list_volumes(self, req): + self._auth(req) + return self._page(req, list(self.volumes.values())) + + @route("POST", _R + "/volumes") + def create_volume(self, req): + self._auth(req) + body = req.json() or {} + name, size, pool_id = str(body.get("name") or ""), body.get("size"), body.get("pool_id") + if not isinstance(size, int) or size <= 0 or size % 512: + raise _err(400, "BAD_REQUEST", "size must be a positive multiple of 512 bytes") + if pool_id not in self.pools: + raise _err(404, "POOL_NOT_FOUND", f"Pool {pool_id} not found") + if body.get("provtype", "THIN") not in ("THIN", "THICK"): + raise _err(400, "BAD_REQUEST", "provtype must be THIN or THICK") + if any(v["name"] == name for v in self.volumes.values()): + raise _err(409, "VOLUME_NAME_CONFLICT", f"Volume name {name} already exists") + vid = self.add_volume(name, size, pool_id) + self.volumes[vid]["provtype"] = body.get("provtype", "THIN") + return _ok(self.volumes[vid], 201) + + @route("GET", _R + "/volumes/" + _ID) + def get_volume(self, req, oid): + self._auth(req) + return _ok(self._vol(int(oid))) + + @route("PUT", _R + "/volumes/" + _ID) + def update_volume(self, req, oid): + self._auth(req) + v = self._vol(int(oid)) + body = req.json() or {} + if "size" in body: + size = body["size"] + if not isinstance(size, int) or size <= 0 or size % 512: + raise _err(400, "BAD_REQUEST", "size must be a positive multiple of 512 bytes") + if size < v["size"]: + self._approved(req, "Shrinking a volume") + v["size"] = size + return _ok(v) + + @route("DELETE", _R + "/volumes/" + _ID) + def delete_volume(self, req, oid): + self._auth(req) + v = self._vol(int(oid)) + self._approved(req, "Deleting a volume") + if any(m["volume_id"] == v["id"] for m in self.luns): + raise _err(409, "VOLUME_IS_MAPPED", "Volume is mapped; unmap it first") + del self.volumes[v["id"]] + return _ok(v) + + @route("GET", _R + "/volumes/" + _ID + "/luns") + def volume_luns(self, req, oid): + self._auth(req) + vid = self._vol(int(oid))["id"] + return _ok([self._lun_view(m) for m in self.luns if m["volume_id"] == vid]) + + # -- hosts ------------------------------------------------------------------------ + + @route("GET", _R + "/hosts") + def list_hosts(self, req): + self._auth(req) + return self._page(req, [self._host_view(h) for h in self.hosts.values()]) + + @route("POST", _R + "/hosts") + def create_host(self, req): + self._auth(req) + name = (req.json() or {}).get("name") + if not name: + raise _err(400, "BAD_REQUEST", "name is required") + if any(h["name"] == name for h in self.hosts.values()): + raise _err(409, "HOST_NAME_CONFLICT", f"Host name {name} already exists") + hid = self.next_id() + self.hosts[hid] = {"id": hid, "name": name, "ports": [], "security_method": "NONE", + "host_type": "LINUX", "host_cluster_id": 0} + return _ok(self._host_view(self.hosts[hid]), 201) + + @route("DELETE", _R + "/hosts/" + _ID) + def delete_host(self, req, oid): + self._auth(req) + h = self._host(int(oid)) + self._approved(req, "Deleting a host") + if any(m["host_id"] == h["id"] for m in self.luns): + raise _err(409, "HOST_HAS_LUNS", "Host has mapped volumes") + del self.hosts[h["id"]] + return _ok(self._host_view(h)) + + @route("POST", _R + "/hosts/" + _ID + "/ports") + def add_port(self, req, oid): + self._auth(req) + h = self._host(int(oid)) + body = req.json() or {} + ptype = str(body.get("type", "")).upper() + if ptype not in ("ISCSI", "FC", "NVME"): + raise _err(400, "BAD_REQUEST", f"Unsupported port type {ptype}") + address = _port_address(ptype, str(body.get("address", ""))) + for other in self.hosts.values(): + for p in other["ports"]: + if p["type"] == ptype and p["address"] == address: + if other["id"] == h["id"]: + raise _err(409, "PORT_ALREADY_EXISTS", + f"Port {address} already belongs to this host") + raise _err(409, "PORT_ALREADY_BELONGS_TO_HOST", + f"Port {address} belongs to host {other['name']}") + port = {"type": ptype, "address": address, "host_id": h["id"]} + h["ports"].append(port) + return _ok(port, 201) + + @route("POST", _R + "/hosts/" + _ID + "/luns") + def map_lun(self, req, oid): + self._auth(req) + h = self._host(int(oid)) + vid = (req.json() or {}).get("volume_id") + if not isinstance(vid, int): + raise _err(400, "BAD_REQUEST", "volume_id must be an integer") + self._vol(vid) + if any(m["host_id"] == h["id"] and m["volume_id"] == vid for m in self.luns): + raise _err(409, "MAPPING_ALREADY_EXISTS", "Volume already mapped to host") + used = {m["lun"] for m in self.luns if m["host_id"] == h["id"]} + lun = next(n for n in range(1, 4096) if n not in used) + m = {"id": self.next_id(), "host_id": h["id"], "volume_id": vid, "lun": lun} + self.luns.append(m) + self.volumes[vid]["mapped"] = True + return _ok(self._lun_view(m), 201) + + @route("DELETE", _R + "/hosts/" + _ID + r"/luns/volume_id/(?P\d+)") + def unmap_lun(self, req, oid, vid): + self._auth(req) + h = self._host(int(oid)) + self._approved(req, "Unmapping a volume") + match = [m for m in self.luns if m["host_id"] == h["id"] and m["volume_id"] == int(vid)] + if not match: + raise _err(404, "LUN_NOT_FOUND", f"Volume {vid} is not mapped to host {h['id']}") + self.luns.remove(match[0]) + if int(vid) in self.volumes: + self.volumes[int(vid)]["mapped"] = any( + m["volume_id"] == int(vid) for m in self.luns) + return _ok(self._lun_view(match[0])) + + def not_routed(self, req: Request) -> Reply: + return Reply(404, {"result": None, "error": { + "code": "NOT_FOUND", "message": f"fake: no route for {req.method} {req.path}"}, + "metadata": {"ready": True}}) + + +def contract_setup() -> ContractSetup: + fake = FakeInfiniBox(username="admin", password="ibox-pass") + return ContractSetup( + fake=fake, + config=make_config("infinidat-infinibox", "https://ibox.fake.example", pool="sb-pool"), + credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("ibox-pass")}), + bad_credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + }, + create_foreign_volume=lambda name, size: str(fake.add_volume(name, size)), + ) diff --git a/simplyblock_core/storage_arrays/testing/lio.py b/simplyblock_core/storage_arrays/testing/lio.py new file mode 100644 index 0000000000..482e7ea7eb --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/lio.py @@ -0,0 +1,335 @@ +"""The data path of a mock array: a vendor fake's volumes, hosts, and +mappings mirrored into the Linux kernel's iSCSI target (LIO). + +The fakes answer the vendor's management API and keep the array's state in +memory, which is enough to test the adapters but presents nothing to a host. +With a mirror attached, every volume of the fake becomes a real block device +exported over iSCSI, every host's initiator an ACL of the target, and every +mapping a LUN of that ACL. A host logged in to the target therefore sees a LUN +appear after a mapping and a SCSI rescan, lose it when it is unmapped (the +fence of a takeover), and see it grow after a resize, exactly as with an +array. This is what lets the control plane's whole multi-attach path (node +add, standby presentation, fencing, takeover, growth) run against a mock. + +The mirror reads the fake through the vendor's own adapter, in process, so a +LUN's size and identity are what the array plug-in reports for it. The +identity reaches the host as the LUN's SCSI unit serial (lsblk SERIAL), which +LUN matching accepts beside the WWN. Only the volume ids and the hosts' +initiator names are read from the fake directly (``lio_volume_ids``, +``lio_host_iqns``, ``lio_all_iqns``). + +Backends: :class:`MemoryLio` for tests, :class:`ConfigfsLio` for the kernel +target (root, ``target_core_iblock``, ``iscsi_target_mod``, and +``python3-rtslib-fb``). Volumes are sparse files on loop devices, exported as +block backstores, so a resize is a truncate and a loop capacity refresh. +""" +from __future__ import annotations + +import hashlib +import logging +import os +import subprocess +import threading +from collections.abc import Iterable +from dataclasses import dataclass, field +from typing import Any +from typing import Protocol as TypingProtocol + +from simplyblock_core.credentials import Credential + +from ..base import ArrayConfig, Protocol +from ..client import ArrayClient + +logger = logging.getLogger(__name__) + +KEY_PREFIX = "sbm_" + + +@dataclass +class LioVolume: + size_bytes: int + serial: str + + +class LioBackend(TypingProtocol): + def list_volumes(self) -> dict[str, LioVolume]: ... + def create_volume(self, key: str, size_bytes: int, serial: str) -> None: ... + def resize_volume(self, key: str, size_bytes: int) -> None: ... + def delete_volume(self, key: str) -> None: ... + def list_acls(self) -> dict[str, dict[int, str]]: ... + def ensure_acl(self, iqn: str) -> None: ... + def delete_acl(self, iqn: str) -> None: ... + def map_lun(self, iqn: str, mapped_lun: int, key: str) -> None: ... + def unmap_lun(self, iqn: str, mapped_lun: int) -> None: ... + def close(self) -> None: ... + + +@dataclass +class MemoryLio: + """The target as dictionaries: volumes by key, and per ACL its LUNs.""" + + volumes: dict[str, LioVolume] = field(default_factory=dict) + acls: dict[str, dict[int, str]] = field(default_factory=dict) + #: Count of changes, for idempotence checks. + ops: int = 0 + + def list_volumes(self) -> dict[str, LioVolume]: + return {k: LioVolume(v.size_bytes, v.serial) for k, v in self.volumes.items()} + + def create_volume(self, key: str, size_bytes: int, serial: str) -> None: + self.ops += 1 + self.volumes[key] = LioVolume(size_bytes, serial) + + def resize_volume(self, key: str, size_bytes: int) -> None: + self.ops += 1 + self.volumes[key].size_bytes = size_bytes + + def delete_volume(self, key: str) -> None: + self.ops += 1 + assert all(key not in luns.values() for luns in self.acls.values()), "volume still mapped" + del self.volumes[key] + + def list_acls(self) -> dict[str, dict[int, str]]: + return {iqn: dict(luns) for iqn, luns in self.acls.items()} + + def ensure_acl(self, iqn: str) -> None: + if iqn not in self.acls: + self.ops += 1 + self.acls[iqn] = {} + + def delete_acl(self, iqn: str) -> None: + self.ops += 1 + del self.acls[iqn] + + def map_lun(self, iqn: str, mapped_lun: int, key: str) -> None: + self.ops += 1 + assert key in self.volumes + self.acls[iqn][mapped_lun] = key + + def unmap_lun(self, iqn: str, mapped_lun: int) -> None: + self.ops += 1 + del self.acls[iqn][mapped_lun] + + def close(self) -> None: + pass + + +class _StaticCredentials: + def __init__(self, credential: Credential): + self._credential = credential + + def resolve(self, ref, context): + return self._credential + + def invalidate(self, ref=None): + pass + + +def _host_identity(vol) -> str: + """What the array plug-in reports for a SCSI LUN (array.san.host_wwn).""" + from simplyblock_core.array.san import host_wwn + return host_wwn(vol, Protocol.ISCSI) + + +class LioMirror: + """Keeps a backend in step with a vendor fake. ``reconcile`` is cheap and + idempotent, and is called after every request the mock server answers.""" + + def __init__(self, fake: Any, config: ArrayConfig, credential: Credential, backend: LioBackend, + *, transport: Any, session_kwargs: dict[str, Any] | None = None, + allow: Iterable[str] = ()): + self.fake = fake + self.backend = backend + #: Initiators given an ACL whether or not a host of the array has them: + #: the hosts log in to the target before they are registered, as they + #: would to an array that presents nothing to an unknown initiator. + self.allow = [i.lower() for i in allow] + self.client = ArrayClient(config, _StaticCredentials(credential), # type: ignore[arg-type] + transport=transport, session_kwargs=session_kwargs) + self._lock = threading.Lock() + + @staticmethod + def key(volume_id: str) -> str: + return KEY_PREFIX + hashlib.sha1(volume_id.encode()).hexdigest()[:12] + + def close(self) -> None: + self.client.close() + self.backend.close() + + # -- desired state ------------------------------------------------------ + + def desired(self) -> tuple[dict[str, LioVolume], dict[str, dict[int, str]]]: + volumes: dict[str, LioVolume] = {} + acls: dict[str, dict[int, str]] = {iqn: {} for iqn in self.allow} + for iqn in self.fake.lio_all_iqns(): + acls.setdefault(iqn, {}) + for vid in self.fake.lio_volume_ids(): + vol = self.client.get_volume(vid) + if vol is None: + continue + key = self.key(vol.id) + volumes[key] = LioVolume(vol.size_bytes, _host_identity(vol)) + for m in self.client.list_mappings(vol.id): + for iqn in self.fake.lio_host_iqns(m.host_id): + luns = acls.setdefault(iqn, {}) + lun = m.lun if m.lun is not None and m.lun not in luns else _free(luns) + luns[lun] = key + return volumes, acls + + # -- reconcile ---------------------------------------------------------- + + def reconcile(self) -> None: + with self._lock: + want_vols, want_acls = self.desired() + have_vols = self.backend.list_volumes() + have_acls = self.backend.list_acls() + + # Unmap first: what is fenced is fenced before anything else moves. + for iqn, luns in have_acls.items(): + want = want_acls.get(iqn, {}) + for n, key in luns.items(): + if want.get(n) != key: + self.backend.unmap_lun(iqn, n) + if iqn not in want_acls: + self.backend.delete_acl(iqn) + for key in have_vols.keys() - want_vols.keys(): + self.backend.delete_volume(key) + for key, vol in want_vols.items(): + have = have_vols.get(key) + if have is None: + self.backend.create_volume(key, vol.size_bytes, vol.serial) + elif vol.size_bytes > have.size_bytes: + self.backend.resize_volume(key, vol.size_bytes) + have_acls = self.backend.list_acls() + for iqn, luns in want_acls.items(): + self.backend.ensure_acl(iqn) + current = have_acls.get(iqn, {}) + for n, key in luns.items(): + if current.get(n) != key: + self.backend.map_lun(iqn, n, key) + + +def _free(luns: dict[int, str]) -> int: + n = 0 + while n in luns: + n += 1 + return n + + +# -- kernel target -------------------------------------------------------------- + + +class ConfigfsLio: + """The kernel's iSCSI target through rtslib. One target, one portal group, + one ACL per initiator; volumes are sparse files on loop devices.""" + + def __init__(self, target_iqn: str, data_dir: str, + portal: tuple[str, int] | list[tuple[str, int]] = ("0.0.0.0", 3260)): + # rtslib keeps its saved configuration here, and refuses to start on a + # host where no target ever ran and the directory is missing. + os.makedirs("/etc/rtslib-fb-target", exist_ok=True) + # python3-rtslib-fb, on the mock's host only + import rtslib_fb as rts # type: ignore[import-not-found] + + self.rts = rts + self.data_dir = data_dir + os.makedirs(data_dir, exist_ok=True) + fabric = rts.FabricModule("iscsi") + self._remove_previous(fabric, target_iqn) + self.target = rts.Target(fabric, target_iqn) + self.tpg = rts.TPG(self.target, 1) + for name, value in (("authentication", "0"), ("generate_node_acls", "0"), + ("demo_mode_write_protect", "0"), ("demo_mode_discovery", "1")): + self.tpg.set_attribute(name, value) + # Several portals give every host several paths to every LUN: the + # multipath case of FC and iSCSI arrays. + for ip, port in (portal if isinstance(portal, list) else [portal]): + rts.NetworkPortal(self.tpg, ip, port) + self.tpg.enable = True + + def _remove_previous(self, fabric, target_iqn: str) -> None: + for target in list(fabric.targets): + if target.wwn == target_iqn: + target.delete() + for so in list(self.rts.RTSRoot().storage_objects): + if so.name.startswith(KEY_PREFIX): + self._delete_object(so) + + def _path(self, key: str) -> str: + return os.path.join(self.data_dir, key + ".img") + + def _object(self, key: str): + for so in self.rts.RTSRoot().storage_objects: + if so.name == key: + return so + raise KeyError(key) + + def _tpg_lun(self, key: str): + for lun in self.tpg.luns: + if lun.storage_object.name == key: + return lun + raise KeyError(key) + + def _delete_object(self, so) -> None: + dev = getattr(so, "udev_path", "") or "" + so.delete() + if dev.startswith("/dev/loop"): + subprocess.run(["losetup", "-d", dev], check=False) + path = self._path(so.name) + if os.path.exists(path): + os.unlink(path) + + def list_volumes(self) -> dict[str, LioVolume]: + out = {} + for so in self.rts.RTSRoot().storage_objects: + if so.name.startswith(KEY_PREFIX): + out[so.name] = LioVolume(os.path.getsize(self._path(so.name)), so.wwn) + return out + + def create_volume(self, key: str, size_bytes: int, serial: str) -> None: + path = self._path(key) + with open(path, "ab") as f: + f.truncate(size_bytes) + dev = subprocess.run(["losetup", "--find", "--show", "--direct-io=on", path], + check=True, capture_output=True, text=True).stdout.strip() + so = self.rts.BlockStorageObject(key, dev=dev, wwn=serial) + self.rts.LUN(self.tpg, storage_object=so) + logger.info("LIO: volume %s (%d bytes, serial %s) on %s", key, size_bytes, serial, dev) + + def resize_volume(self, key: str, size_bytes: int) -> None: + so = self._object(key) + with open(self._path(key), "ab") as f: + f.truncate(size_bytes) + subprocess.run(["losetup", "-c", so.udev_path], check=True) + logger.info("LIO: volume %s grown to %d bytes", key, size_bytes) + + def delete_volume(self, key: str) -> None: + self._tpg_lun(key).delete() + self._delete_object(self._object(key)) + logger.info("LIO: volume %s deleted", key) + + def list_acls(self) -> dict[str, dict[int, str]]: + return {acl.node_wwn: {ml.mapped_lun: ml.tpg_lun.storage_object.name for ml in acl.mapped_luns} + for acl in self.tpg.node_acls} + + def _acl(self, iqn: str): + return self.rts.NodeACL(self.tpg, iqn, mode="lookup") + + def ensure_acl(self, iqn: str) -> None: + self.rts.NodeACL(self.tpg, iqn) + + def delete_acl(self, iqn: str) -> None: + self._acl(iqn).delete() + + def map_lun(self, iqn: str, mapped_lun: int, key: str) -> None: + self.rts.MappedLUN(self._acl(iqn), mapped_lun, self._tpg_lun(key)) + logger.info("LIO: %s mapped to %s as LUN %d", key, iqn, mapped_lun) + + def unmap_lun(self, iqn: str, mapped_lun: int) -> None: + for ml in self._acl(iqn).mapped_luns: + if ml.mapped_lun == mapped_lun: + ml.delete() + logger.info("LIO: LUN %d of %s unmapped", mapped_lun, iqn) + + def close(self) -> None: + pass diff --git a/simplyblock_core/storage_arrays/testing/lio_selftest.py b/simplyblock_core/storage_arrays/testing/lio_selftest.py new file mode 100644 index 0000000000..b7730bb3cd --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/lio_selftest.py @@ -0,0 +1,485 @@ +"""End-to-end self-test of an array plug-in against a mock array with a real +data path, on one Linux host. + +The mock array (a vendor fake with its LIO mirror, ``testing.lio``) is served +over HTTPS, and two iSCSI interfaces of this host, with two initiator names, +play two storage nodes. The array plug-in of the multi-attach mode +(``simplyblock_core.array.san``) drives the mock exactly as the control plane +does, and the node agent's own functions (``scan_for_new_luns``, +``find_lun_devices``, ``rescan_block_device``) find the LUNs on the host. The +checks are about the data path, on real block devices: + +1. a LUN created and attached to host A appears on A after a scan; +2. attached to host B as well, B reads the data A wrote (multi-attach); +3. fenced from A, A's I/O fails while B's goes on; +4. grown, the new size is seen by B after a rescan; +5. unfenced, A sees the LUN again; +6. deleted, it is gone from the target. + +Run as root on a host with LIO (``target_core_iblock``, ``iscsi_target_mod``), +open-iscsi, and python3-rtslib-fb:: + + python -m simplyblock_core.storage_arrays.testing.lio_selftest --vendor pure-flasharray + +On WSL, the portal goes on a dummy interface, because WSL's loopback refuses +the kernel target's listener:: + + ... lio_selftest --vendor pure-flasharray --portal 192.168.250.1:3260 --dummy-portal +""" +from __future__ import annotations + +import argparse +import hashlib +import importlib +import os +import subprocess +import sys +import threading +import time + +from ..registry import ADAPTERS, canonical_vendor +from . import FakeTransport, serve +from .lio import ConfigfsLio, LioMirror + +GIB = 1 << 30 +TARGET = "iqn.2026-11.io.simplyblock:selftest" +IQN_A = "iqn.2026-11.io.simplyblock:selftest-host-a" +IQN_B = "iqn.2026-11.io.simplyblock:selftest-host-b" +PATTERN_BYTES = 4 << 20 + + +def sh(*cmd: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run(list(cmd), check=check, capture_output=True, text=True) + + +def step(text: str) -> None: + print(f"--- {text}", flush=True) + + +class Hosts: + """Two iSCSI interfaces of this host, logged in to the mock's target.""" + + def __init__(self, portal: str): + self.portal = portal + if sh("pgrep", "-x", "iscsid", check=False).returncode != 0: + # Socket-activated under systemd; started by hand elsewhere. + if sh("systemctl", "start", "iscsid", check=False).returncode != 0: + subprocess.run(["iscsid"], check=True) + time.sleep(1) + for iface, iqn in (("sbtest-a", IQN_A), ("sbtest-b", IQN_B)): + sh("iscsiadm", "-m", "iface", "-I", iface, "-o", "new", check=False) + sh("iscsiadm", "-m", "iface", "-I", iface, "-o", "update", "-n", "iface.initiatorname", "-v", iqn) + sh("iscsiadm", "-m", "discovery", "-t", "st", "-p", portal, "-I", "sbtest-a", "-I", "sbtest-b") + sh("iscsiadm", "-m", "node", "-T", TARGET, "-p", portal, "--login") + + def sid_of(self, iqn: str) -> str: + """The SCSI host (``hostN``) of the session with initiator ``iqn``.""" + for sess in os.listdir("/sys/class/iscsi_session"): + base = f"/sys/class/iscsi_session/{sess}" + if open(f"{base}/initiatorname").read().strip() == iqn: + # .../hostN/sessionM: the session's SCSI host is in its path. + path = os.path.realpath(f"{base}/device") + return next(p for p in path.split("/") if p.startswith("host")) + raise RuntimeError(f"no iSCSI session of {iqn}") + + def close(self) -> None: + sh("iscsiadm", "-m", "node", "-T", TARGET, "-p", self.portal, "--logout", check=False) + sh("iscsiadm", "-m", "node", "-T", TARGET, "-p", self.portal, "-o", "delete", check=False) + for iface in ("sbtest-a", "sbtest-b"): + sh("iscsiadm", "-m", "iface", "-I", iface, "-o", "delete", check=False) + + +def device_of(node_utils, wwn: str, scsi_host: str, timeout: float = 20.0) -> str | None: + """The block device of LUN ``wwn`` behind ``scsi_host``, after scans.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + node_utils.scan_for_new_luns() + subprocess.run(["udevadm", "settle"], check=False) + for d in node_utils.find_lun_devices(wwn): + link = os.path.realpath(f"/sys/block/{d['name']}/device") + if f"/{scsi_host}/" in link: + return d["device_path"] + time.sleep(1) + return None + + +def write_pattern(dev: str) -> str: + flags = os.O_WRONLY + if sys.platform == "linux": + flags |= os.O_DIRECT | os.O_SYNC + data = os.urandom(PATTERN_BYTES) + fd = os.open(dev, flags) + try: + import mmap + buf = mmap.mmap(-1, PATTERN_BYTES) + buf.write(data) + os.write(fd, buf) + finally: + os.close(fd) + return hashlib.sha256(data).hexdigest() + + +def read_pattern(dev: str) -> str: + """O_DIRECT read into a page-aligned buffer, as SPDK reads the device.""" + import mmap + flags = os.O_RDONLY + if sys.platform == "linux": + flags |= os.O_DIRECT + buf = mmap.mmap(-1, PATTERN_BYTES) + with os.fdopen(os.open(dev, flags), "rb", buffering=0) as f: + n = f.readinto(buf) + if n != PATTERN_BYTES: + raise OSError(f"short read of {dev}: {n} bytes") + return hashlib.sha256(buf[:]).hexdigest() + + +MODULES = ("configfs", "target_core_mod", "iscsi_target_mod", "target_core_iblock", "iscsi_tcp", "loop", + "dm_multipath") + + +MULTIPATH_CONF = """# written by the simplyblock LIO self-test; restored afterwards +defaults { + user_friendly_names yes + find_multipaths yes + polling_interval 1 +} +# LIO's built-in device entry queues for 12 checks; a fenced map fails at once. +overrides { + no_path_retry fail +} +blacklist { + device { + vendor ".*" + product ".*" + } +} +blacklist_exceptions { + device { + vendor "LIO-ORG" + product ".*" + } +} +""" + + +class Multipathd: + """multipathd with a configuration that claims only the mock's LUNs, for + the duration of the multipath scenario.""" + + CONF = "/etc/multipath.conf" + BACKUP = "/etc/multipath.conf.sb-selftest" + + def __init__(self) -> None: + if os.path.exists(self.CONF): + os.replace(self.CONF, self.BACKUP) + with open(self.CONF, "w") as f: + f.write(MULTIPATH_CONF) + sh("systemctl", "start", "multipathd", check=False) + self.direct = sh("systemctl", "is-active", "multipathd", check=False).stdout.strip() != "active" + if self.direct: + # systemd skips the unit where it detects a container (WSL). + sh("multipathd") + for _ in range(20): + if sh("multipathd", "show", "daemon", check=False).returncode == 0: + break + time.sleep(0.5) + sh("multipathd", "reconfigure", check=False) + + def close(self) -> None: + sh("multipath", "-F", check=False) + if self.direct: + sh("multipathd", "shutdown", check=False) + else: + sh("systemctl", "stop", "multipathd", "multipathd.socket", check=False) + os.unlink(self.CONF) + if os.path.exists(self.BACKUP): + os.replace(self.BACKUP, self.CONF) + + +def map_of(node_utils, wwn: str, paths: int, timeout: float = 30.0) -> dict | None: + """The multipath map of LUN ``wwn`` once it has ``paths`` paths.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + node_utils.scan_for_new_luns() + subprocess.run(["udevadm", "settle"], check=False) + for d in node_utils.find_lun_devices(wwn): + if d.get("type") == "mpath": + live: dict = next((x for x in node_utils.get_block_devices_info() if x["name"] == d["name"]), {}) + if len(live.get("paths", [])) >= paths: + return d + time.sleep(1) + return None + + +def run_multipath(args, setup, vendor: str, portals: list[str], plugin_cls, node_utils, mirror, backend, + server, check) -> None: + """One host reaching every LUN over two portals, under dm-multipath.""" + step("multipath: host A logs in over two portals") + sh("iscsiadm", "-m", "iface", "-I", "sbtest-a", "-o", "new", check=False) + sh("iscsiadm", "-m", "iface", "-I", "sbtest-a", "-o", "update", "-n", "iface.initiatorname", "-v", IQN_A) + for portal in portals: + sh("iscsiadm", "-m", "discovery", "-t", "st", "-p", portal, "-I", "sbtest-a") + sh("iscsiadm", "-m", "node", "-T", TARGET, "--login") + mpd = Multipathd() + lun = None + plugin = None + try: + endpoint = f"https://127.0.0.1:{args.port}" + options = dict(setup.config.options, protocol="iscsi", ca_bundle=open(server.certfile).read()) + plugin = plugin_cls(endpoint=endpoint, options=options, cluster_id="selftest") + plugin.connect(endpoint, {n: setup.credential.get(n) for n in setup.credential.field_names}, options) + host_a = plugin.ensure_host("selftest-a", {"iscsi_iqn": IQN_A}) + + step("M1. create and attach: the LUN is a multipath map with two paths") + lun = plugin.create_lun("selftest-mpath", GIB, {}) + plugin.attach(lun["id"], host_a) + m = map_of(node_utils, lun["wwn"], paths=2) + check(m is not None, f"LUN {lun['wwn']} found as a multipath map ({m and m['device_path']})") + if m is None: + print(sh("systemctl", "is-active", "multipathd", check=False).stdout) + r = sh("multipath", "-v3", check=False) + print("\n".join((r.stdout + r.stderr).splitlines()[-40:])) + print(sh("multipath", "-ll", check=False).stdout) + print(sh("lsblk", "-o", "NAME,KNAME,TYPE,SIZE,SERIAL,WWN,VENDOR", check=False).stdout) + return + dev = m["device_path"] + digest = write_pattern(dev) + try: + check(read_pattern(dev) == digest, "the map reads back what was written over both paths") + except OSError as e: + check(False, f"the map reads back what was written over both paths ({e})") + + step("M2. losing a path does not stop the I/O") + sh("iscsiadm", "-m", "node", "-T", TARGET, "-p", portals[1], "-I", "sbtest-a", "--logout") + errors: list[str] = [] + got = "" + for _ in range(10): + try: + got = read_pattern(dev) + break + except OSError as e: + errors.append(str(e)) + small = subprocess.run(["dd", f"if={dev}", "bs=4k", "count=1", "iflag=direct", + "status=none"], capture_output=True) + errors.append(f"4k read rc={small.returncode} {small.stderr.decode(errors='replace').strip()}") + time.sleep(1) + print(f" reads before success: {len(errors) // 2}, errors: {errors[:4]}") + check(got == digest, "the map reads the data over the remaining path") + if got != digest: + print(sh("multipath", "-ll", check=False).stdout) + print(sh("cat", f"/sys/block/{m['kname']}/queue/max_sectors_kb", + f"/sys/block/{m['kname']}/queue/logical_block_size", check=False).stdout) + for what, cmd in (("path direct", ["dd", "if=/dev/sde", "bs=4k", "count=1", "iflag=direct", + "of=/dev/null"]), + ("map buffered", ["dd", f"if={dev}", "bs=4k", "count=1", "of=/dev/null"]), + ("dm table", ["dmsetup", "table", m["name"]]), + ("dm status", ["dmsetup", "status", m["name"]])): + r = sh(*cmd, check=False) + print(f" {what}: rc={r.returncode} {(r.stdout + r.stderr).strip()[:300]}") + print("\n".join(sh("dmesg", check=False).stdout.splitlines()[-8:])) + return + sh("iscsiadm", "-m", "node", "-T", TARGET, "-p", portals[1], "-I", "sbtest-a", "--login") + check(map_of(node_utils, lun["wwn"], paths=2) is not None, "the path is back in the map") + + step("M3. grow") + plugin.resize(lun["id"], 2 * GIB) + grown = node_utils.rescan_block_device(dev) + check(grown["size"] >= 2 * GIB and any(r.startswith("multipathd:") for r in grown["rescanned"]), + f"the map is {grown['size']} bytes after rescanning its paths and resizing it") + + step("M4. fence: the map's I/O fails") + plugin.fence(lun["id"], host_a) + failed = "" + for _ in range(15): + try: + read_pattern(dev) + except OSError as e: + failed = str(e) + break + time.sleep(1) + check(bool(failed), f"the map's I/O fails after the fence ({failed[:60]})") + + step("M5. unfence: the map serves again") + plugin.unfence(lun["id"], host_a) + sh("iscsiadm", "-m", "session", "--rescan", check=False) + back = "" + for _ in range(30): + sh("multipathd", "reconfigure", check=False) + try: + back = read_pattern(dev) + break + except OSError: + time.sleep(1) + check(back == digest, "the map reads the data again") + + step("M6. delete") + sh("multipath", "-f", m["name"], check=False) + plugin.delete_lun(lun["id"]) + check(mirror.key(lun["id"]) not in backend.list_volumes(), "the LUN is gone from the target") + lun = None + finally: + if plugin is not None and lun is not None: + try: + plugin.delete_lun(lun["id"]) + except Exception as e: # noqa: BLE001 + print(f"cleanup: {e}") + mpd.close() + sh("iscsiadm", "-m", "node", "-T", TARGET, "--logout", check=False) + sh("iscsiadm", "-m", "node", "-T", TARGET, "-o", "delete", check=False) + sh("iscsiadm", "-m", "iface", "-I", "sbtest-a", "-o", "delete", check=False) + + +def prepare_host(portal_ip: str, dummy_portal: bool) -> None: + """Load the target and initiator modules and mount configfs (all + idempotent). With ``dummy_portal`` the portal address is put on a dummy + interface: hosts whose loopback refuses kernel listeners (WSL's mirrored + networking) need it.""" + for module in MODULES: + sh("modprobe", module) + if sh("mountpoint", "-q", "/sys/kernel/config", check=False).returncode != 0: + sh("mount", "-t", "configfs", "none", "/sys/kernel/config") + if dummy_portal: + if sh("ip", "link", "show", "sbdummy", check=False).returncode != 0: + sh("ip", "link", "add", "sbdummy", "type", "dummy") + sh("ip", "link", "set", "sbdummy", "up") + if f" {portal_ip}/" not in sh("ip", "-o", "addr", "show", "dev", "sbdummy").stdout: + sh("ip", "addr", "add", f"{portal_ip}/32", "dev", "sbdummy") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--vendor", default="pure-flasharray", help=", ".join(sorted(ADAPTERS))) + parser.add_argument("--port", type=int, default=8443) + parser.add_argument("--portal", default="127.0.0.1:3260") + parser.add_argument("--lio-dir", default="/var/tmp/sb-lio-selftest") + parser.add_argument("--dummy-portal", action="store_true", + help="put the portal address on a dummy interface (WSL)") + parser.add_argument("--multipath", metavar="IP:PORT", + help="run the multipath scenario: a second portal, and one host under dm-multipath " + "(instead of the two-host scenario)") + args = parser.parse_args() + + if sys.platform != "linux": + print("runs on Linux only", file=sys.stderr) + return 2 + if os.geteuid() != 0: + print("run as root", file=sys.stderr) + return 2 + from simplyblock_core import array + from simplyblock_web import node_utils + + vendor = canonical_vendor(args.vendor) + module = importlib.import_module(f"{__package__}.{ADAPTERS[vendor].split(':')[0].split('.')[-1]}") + setup = module.contract_setup() + ip, _, port = args.portal.rpartition(":") + prepare_host(ip, args.dummy_portal) + portals = [(ip, int(port))] + if args.multipath: + ip2, _, port2 = args.multipath.rpartition(":") + prepare_host(ip2, args.dummy_portal) + portals.append((ip2, int(port2))) + backend = ConfigfsLio(TARGET, args.lio_dir, portals) + mirror = LioMirror(setup.fake, setup.config, setup.credential, backend, + transport=FakeTransport(setup.fake), allow=[IQN_A, IQN_B]) + mirror.reconcile() + server = serve(setup.fake, "127.0.0.1", args.port, after_request=mirror.reconcile) + threading.Thread(target=server.serve_forever, daemon=True).start() + + failures: list[str] = [] + + def check(ok: bool, what: str) -> None: + print(("PASS " if ok else "FAIL ") + what, flush=True) + if not ok: + failures.append(what) + + if args.multipath: + try: + run_multipath(args, setup, vendor, [args.portal, args.multipath], array.plugin_class(vendor), + node_utils, mirror, backend, server, check) + finally: + server.shutdown() + backend_cleanup(backend) + print(f"{'FAILED' if failures else 'PASSED'}: {len(failures)} failure(s)", flush=True) + return 1 if failures else 0 + + hosts = Hosts(args.portal) + + plugin = None + lun = None + try: + endpoint = f"https://127.0.0.1:{args.port}" + options = dict(setup.config.options, protocol="iscsi", ca_bundle=open(server.certfile).read()) + plugin = array.plugin_class(vendor)(endpoint=endpoint, options=options, cluster_id="selftest") + plugin.connect(endpoint, {n: setup.credential.get(n) for n in setup.credential.field_names}, options) + host_a = plugin.ensure_host("selftest-a", {"iscsi_iqn": IQN_A}) + host_b = plugin.ensure_host("selftest-b", {"iscsi_iqn": IQN_B}) + scsi_a, scsi_b = hosts.sid_of(IQN_A), hosts.sid_of(IQN_B) + + step("1. create and attach to host A") + lun = plugin.create_lun("selftest-lun", GIB, {}) + plugin.attach(lun["id"], host_a) + dev_a = device_of(node_utils, lun["wwn"], scsi_a) + check(dev_a is not None, f"LUN {lun['wwn']} appears on host A ({dev_a})") + if dev_a is None: + return 1 + digest = write_pattern(dev_a) + + step("2. attach to host B as well (multi-attach)") + plugin.attach(lun["id"], host_b) + dev_b = device_of(node_utils, lun["wwn"], scsi_b) + check(dev_b is not None, f"LUN appears on host B ({dev_b})") + if dev_b is None: + return 1 + check(read_pattern(dev_b) == digest, "host B reads what host A wrote") + check(sorted(plugin.get(lun["id"])["hosts"]) == sorted([host_a, host_b]), + "the array reports both hosts") + + step("3. fence host A") + plugin.fence(lun["id"], host_a) + try: + read_pattern(dev_a) + check(False, "host A's I/O fails after the fence") + except OSError as e: + check(True, f"host A's I/O fails after the fence ({str(e)[:60]})") + check(read_pattern(dev_b) == digest, "host B still reads the data") + + step("4. grow") + plugin.resize(lun["id"], 2 * GIB) + grown = node_utils.rescan_block_device(dev_b) + check(grown["size"] >= 2 * GIB, f"host B sees {grown['size']} bytes after a rescan") + + step("5. unfence host A") + plugin.unfence(lun["id"], host_a) + sh("iscsiadm", "-m", "session", "--rescan", check=False) + dev_a2 = device_of(node_utils, lun["wwn"], scsi_a) + check(dev_a2 is not None and read_pattern(dev_a2) == digest, "host A reads the data again") + + step("6. delete") + plugin.delete_lun(lun["id"]) + check(mirror.key(lun["id"]) not in backend.list_volumes(), "the LUN is gone from the target") + lun = None + finally: + if plugin is not None and lun is not None: + try: + plugin.delete_lun(lun["id"]) + except Exception as e: # noqa: BLE001 + print(f"cleanup: {e}") + hosts.close() + server.shutdown() + backend_cleanup(backend) + print(f"{'FAILED' if failures else 'PASSED'}: {len(failures)} failure(s)", flush=True) + return 1 if failures else 0 + + +def backend_cleanup(backend: ConfigfsLio) -> None: + for key in list(backend.list_volumes()): + for iqn, luns in backend.list_acls().items(): + for n, k in luns.items(): + if k == key: + backend.unmap_lun(iqn, n) + backend.delete_volume(key) + backend.target.delete() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/simplyblock_core/storage_arrays/testing/netapp_ontap.py b/simplyblock_core/storage_arrays/testing/netapp_ontap.py new file mode 100644 index 0000000000..b6ea286626 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/netapp_ontap.py @@ -0,0 +1,916 @@ +"""Fake NetApp ONTAP 9.x REST API (the subset the adapter uses). + +Modelled behaviour: + +* HTTP basic auth on every request (``401`` + ONTAP error body otherwise). +* Collections answer ``{"records": [...], "num_records": n}``; query + parameters filter on (dotted, array-flattening) fields with ``|`` for OR + and ``*`` wildcards; ``fields=`` projects the records (key fields ``uuid`` + and ``name`` are always returned), so an adapter that forgets to ask for a + field does not get it. +* Errors: ``{"error": {"message": ..., "code": ...}}``; ``404`` for unknown + objects, ``409`` for duplicates, ``400`` for everything else. Codes ``4`` + (entry doesn't exist), ``5374242`` (LUN exists), ``72090025`` (subsystem + exists) and ``5374922`` (LUN map exists) are ONTAP's own, as listed in + Trident ``storage_drivers/ontap/api/abstraction_error.go``; the other + codes are illustrative. +* FlexVol create/resize/delete are asynchronous: ``202`` with a job link; + the effect happens when the job runs, after ``job_polls`` polls of + ``/api/cluster/jobs/{uuid}``. With ``async_san`` LUN and namespace + create/modify/delete answer the same way (the ONTAP spec allows ``202`` + for them). +* FlexVol ``DELETE`` without ``force=true`` moves the volume to + :attr:`FakeNetAppOntap.recovery_queue`. +* An igroup's ``protocol`` cannot be changed after creation. +* ``GET /api/cluster`` returns ``disaggregated`` (ASA r2) only when asked for + in ``fields``, as for SVM-scoped accounts. +* Size rules: FlexVol >= 20 MiB (rounded up to 4 KiB); LUN/namespace must fit + in its FlexVol (usable = size minus snapshot reserve), cannot shrink, and a + namespace is a multiple of its block size. +* Name rules: FlexVol ``[A-Za-z_][A-Za-z0-9_]*``; LUN/namespace path + ``/vol//``. +* An initiator / host NQN may be in several igroups / subsystems (as on + ONTAP); a namespace can be mapped to one subsystem only; mapped + LUNs/namespaces/igroups/subsystems cannot be deleted. +""" +from __future__ import annotations + +import copy +import fnmatch +import re +import secrets +import string +import uuid as uuid_mod +from collections.abc import Callable +from typing import Any +from urllib.parse import unquote + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +_U = r"(?P[0-9a-fA-F-]{36})" +_RESERVED = frozenset({"fields", "return_records", "return_timeout", "max_records", "order_by", + "allow_delete_while_mapped", "allow_delete_with_hosts"}) +_FLEXVOL_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,202}$") +_PATH_RE = re.compile(r"^/vol/(?P[^/]+)/(?P[A-Za-z0-9_.\-{}]{1,255})$") +_IGROUP_RE = re.compile(r"^[A-Za-z0-9_.:\-]{1,96}$") +_SUBSYS_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.\-]{0,95}$") +_WWPN_RE = re.compile(r"^([0-9a-f]{2}:){7}[0-9a-f]{2}$", re.IGNORECASE) +_SERIAL_CHARS = string.ascii_letters + string.digits + "-]?$" +MIN_VOLUME = 20 << 20 +MAX_LUN = 128 << 40 + + +def _err(status: int, message: str, code: str = "2", target: str | None = None) -> FakeError: + body: dict[str, Any] = {"error": {"message": message, "code": code}} + if target: + body["error"]["target"] = target + return FakeError(status, body) + + +def _not_found(what: str = "entry") -> FakeError: + return _err(404, f"{what} doesn't exist", "4") + + +def _resolve(obj: Any, parts: list[str]) -> list[Any]: + if not parts: + return obj if isinstance(obj, list) else [obj] + if isinstance(obj, list): + return [v for e in obj for v in _resolve(e, parts)] + if isinstance(obj, dict) and parts[0] in obj: + return _resolve(obj[parts[0]], parts[1:]) + return [] + + +def _matches(rec: dict[str, Any], key: str, pattern: str) -> bool: + values = _resolve(rec, key.split(".")) + texts = [str(v).lower() for v in values] # True -> "true", as in ONTAP queries + for p in pattern.split("|"): + if any(fnmatch.fnmatchcase(t, p.lower()) for t in texts): + return True + return False + + +def _project(obj: Any, paths: list[list[str]]) -> Any: + if any(not p for p in paths): + return copy.deepcopy(obj) + if isinstance(obj, list): + return [_project(e, paths) for e in obj] + if not isinstance(obj, dict): + return copy.deepcopy(obj) + out: dict[str, Any] = {} + groups: dict[str, list[list[str]]] = {} + for p in paths: + groups.setdefault(p[0], []).append(p[1:]) + for key, sub in groups.items(): + if key in obj: + out[key] = _project(obj[key], sub) + return out + + +class FakeNetAppOntap(FakeArray): + vendor = "netapp-ontap" + + def __init__(self, username: str = "admin", password: str = "netapp1!", + svm: str = "svm1"): + super().__init__() + self.username = username + self.password = password + self.svms: dict[str, str] = {svm: str(uuid_mod.uuid4())} + self.aggregates: list[dict[str, Any]] = [ + {"uuid": str(uuid_mod.uuid4()), "name": "aggr1", + "space": {"block_storage": {"available": 10 << 40}}}, + {"uuid": str(uuid_mod.uuid4()), "name": "aggr2", + "space": {"block_storage": {"available": 30 << 40}}}, + ] + self.volumes: dict[str, dict[str, Any]] = {} + self.luns: dict[str, dict[str, Any]] = {} + self.namespaces: dict[str, dict[str, Any]] = {} + self.igroups: dict[str, dict[str, Any]] = {} + self.lun_maps: list[dict[str, Any]] = [] + self.subsystems: dict[str, dict[str, Any]] = {} + self.subsystem_maps: list[dict[str, Any]] = [] + self.jobs: dict[str, dict[str, Any]] = {} + #: Number of job polls answering ``running`` before a job runs. + self.job_polls = 1 + #: When True, jobs never finish. + self.jobs_stuck = False + self._job_failures: list[tuple[str, str]] = [] + #: When True, LUN/namespace create/modify/delete answer 202 + job. + self.async_san = False + #: FlexVols deleted without ``force=true``, by uuid. + self.recovery_queue: dict[str, dict[str, Any]] = {} + self.version = (9, 14, 1) + #: ASA r2 personality. + self.disaggregated = False + + # -- fake controls ------------------------------------------------------ + + def set_credential(self, **fields: str) -> None: + self.username = fields["username"] + self.password = fields["password"] + + def fail_next_job(self, message: str, code: str = "917504") -> None: + """Make the next job created end in ``failure`` with this message.""" + self._job_failures.append((message, code)) + + def svm_ref(self, name: str) -> dict[str, str]: + if name not in self.svms: + raise _err(404, f'SVM "{name}" not found', "2621462") + return {"name": name, "uuid": self.svms[name]} + + def add_flexvol(self, name: str, size: int, svm: str = "svm1", + snapshot_reserve: int = 0) -> str: + vid = str(uuid_mod.uuid4()) + self.volumes[vid] = { + "uuid": vid, "name": name, "svm": self.svm_ref(svm), "size": size, + "type": "rw", "style": "flexvol", "state": "online", + "guarantee": {"type": "none"}, + "space": {"snapshot": {"reserve_percent": snapshot_reserve}}, + "snapshot_policy": {"name": "default"}, + "aggregates": [{"name": "aggr1", "uuid": self.aggregates[0]["uuid"]}], + "comment": "", + } + return vid + + def add_lun(self, path: str, size: int, svm: str = "svm1") -> str: + vol = self._vol_of_path(path, svm) + lid = str(uuid_mod.uuid4()) + self.luns[lid] = { + "uuid": lid, "name": path, "svm": self.svm_ref(svm), "os_type": "linux", + "serial_number": "".join(secrets.choice(_SERIAL_CHARS) for _ in range(12)), + "space": {"size": size, "guarantee": {"requested": False, "reserved": False}}, + "location": {"logical_unit": path.rsplit("/", 1)[1], + "volume": {"name": vol["name"], "uuid": vol["uuid"]}}, + "status": {"mapped": False, "state": "online"}, + } + return lid + + def add_foreign_lun(self, name: str, size: int) -> str: + flex = name.replace("-", "_") + self.add_flexvol(flex, size * 2) + return self.add_lun(f"/vol/{flex}/{name}", size) + + # -- auth / plumbing ------------------------------------------------------ + + def _auth(self, req: Request) -> None: + if req.basic_auth() != (self.username, self.password): + raise _err(401, "User is not authorized.", "6") + + def _collection(self, req: Request, records: list[dict[str, Any]], + keys: tuple[str, ...] = ("uuid", "name")) -> Reply: + filters = {k: v[0] for k, v in req.query.items() if k not in _RESERVED and v} + found = [r for r in records if all(_matches(r, k, v) for k, v in filters.items())] + return Reply(200, {"records": [self._fields(r, req, keys) for r in found], + "num_records": len(found), + "_links": {"self": {"href": req.path}}}) + + @staticmethod + def _fields(rec: dict[str, Any], req: Request, keys: tuple[str, ...]) -> dict[str, Any]: + fields = req.q("fields") + if fields == "*": + return copy.deepcopy(rec) + paths = [[k] for k in keys if k in rec] + if fields: + paths += [f.strip().split(".") for f in fields.split(",") if f.strip()] + return _project(rec, paths) + + @staticmethod + def _created(req: Request, rec: dict[str, Any], location: str) -> Reply: + if req.q("return_records") == "true": + return Reply(201, {"num_records": 1, "records": [copy.deepcopy(rec)]}, + {"Location": location}) + return Reply(201, None, {"Location": location}) + + def _svm_of(self, body: dict[str, Any]) -> dict[str, str]: + svm = (body.get("svm") or {}).get("name") + if not svm: + raise _err(400, 'Field "svm.name" is required.', "2") + return self.svm_ref(svm) + + def _vol_of_path(self, path: str, svm: str) -> dict[str, Any]: + m = _PATH_RE.match(path or "") + if not m: + raise _err(400, f'Invalid path "{path}". Paths must be of the form ' + '"/vol//".', "5374082") + for v in self.volumes.values(): + if v["name"] == m["vol"] and v["svm"]["name"] == svm: + return v + raise _err(404, f'Volume "{m["vol"]}" in SVM "{svm}" does not exist.', "917927") + + @staticmethod + def _usable(vol: dict[str, Any]) -> int: + reserve = vol["space"]["snapshot"]["reserve_percent"] + return vol["size"] * (100 - reserve) // 100 + + # -- jobs ----------------------------------------------------------------- + + def _san(self, description: str, run: Callable[[], Reply]) -> Reply: + """LUN/namespace calls: synchronous, or a job when ``async_san``.""" + if not self.async_san: + return run() + + def action() -> None: + run() + + return self._job(description, action) + + def _job(self, description: str, action: Callable[[], None]) -> Reply: + jid = str(uuid_mod.uuid4()) + failure = self._job_failures.pop(0) if self._job_failures else None + self.jobs[jid] = {"uuid": jid, "description": description, "state": "queued", + "message": "", "code": 0, "_polls": self.job_polls, + "_action": action, "_failure": failure} + href = f"/api/cluster/jobs/{jid}" + return Reply(202, {"job": {"uuid": jid, "_links": {"self": {"href": href}}}}) + + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return list(self.luns) + list(self.namespaces) + + def lio_all_iqns(self) -> list[str]: + return sorted({i["name"].lower() for g in self.igroups.values() for i in g["initiators"] if i["name"].lower().startswith("iqn.")}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + for g in self.igroups.values(): + if host_id in (g["uuid"], g["name"]): + return [i["name"].lower() for i in g["initiators"] if i["name"].lower().startswith("iqn.")] + return [] + + @route("GET", r"/api/cluster/jobs/" + _U) + def get_job(self, req, uuid): + self._auth(req) + job = self.jobs.get(uuid) + if job is None: + raise _not_found() + if job["state"] in ("queued", "running"): + if self.jobs_stuck or job["_polls"] > 0: + job["_polls"] -= 1 + job["state"] = "running" + job["message"] = "In progress" + elif job["_failure"]: + job["state"], job["message"] = "failure", job["_failure"][0] + job["code"] = int(job["_failure"][1]) + else: + try: + job["_action"]() + job["state"], job["message"] = "success", "success" + except FakeError as e: + err = e.reply.body["error"] + job["state"], job["message"] = "failure", err["message"] + job["code"] = int(err["code"]) + return Reply(200, {k: v for k, v in job.items() if not k.startswith("_")}) + + # -- cluster ---------------------------------------------------------------- + + @route("GET", "/api/cluster") + def cluster(self, req): + self._auth(req) + gen, major, minor = self.version + rec: dict[str, Any] = { + "name": "fake-ontap", "uuid": "1cd8a442-86d1-11e0-ae1c-123478563412", + "version": {"full": f"NetApp Release {gen}.{major}.{minor}: Mon Jan 01 00:00:00 " + "UTC 2024", "generation": gen, "major": major, + "minor": minor}} + asked = {f.strip() for f in (req.q("fields") or "").split(",")} + if "disaggregated" in asked and self.version >= (9, 16, 1): + rec["disaggregated"] = self.disaggregated + if "san_optimized" in asked and self.version >= (9, 16, 1): + rec["san_optimized"] = self.disaggregated + return Reply(200, rec) + + @route("GET", "/api/cluster/nodes") + def nodes(self, req): + self._auth(req) + return self._collection(req, [ + {"uuid": "6ab1b4b3-0000-0000-0000-000000000001", "name": "fake-ontap-01", + "model": "AFF-A400", "serial_number": "721802000123"}, + {"uuid": "6ab1b4b3-0000-0000-0000-000000000002", "name": "fake-ontap-02", + "model": "AFF-A400", "serial_number": "721802000124"}, + ]) + + @route("GET", "/api/svm/svms") + def get_svms(self, req): + self._auth(req) + return self._collection(req, [{"uuid": u, "name": n} for n, u in self.svms.items()]) + + @route("GET", "/api/storage/aggregates") + def get_aggregates(self, req): + self._auth(req) + return self._collection(req, self.aggregates) + + # -- FlexVols --------------------------------------------------------------- + + @route("GET", "/api/storage/volumes") + def get_volumes(self, req): + self._auth(req) + return self._collection(req, list(self.volumes.values())) + + @route("GET", "/api/storage/volumes/" + _U) + def get_volume(self, req, uuid): + self._auth(req) + if uuid not in self.volumes: + raise _not_found() + return Reply(200, self._fields(self.volumes[uuid], req, ("uuid", "name")) if req.q( + "fields") else copy.deepcopy(self.volumes[uuid])) + + @route("POST", "/api/storage/volumes") + def post_volume(self, req): + self._auth(req) + body = req.json() or {} + svm = self._svm_of(body) + name = body.get("name", "") + if not _FLEXVOL_RE.match(name): + raise _err(400, f'Invalid volume name "{name}". Volume names must begin with a ' + 'letter or underscore and contain only letters, digits and ' + 'underscores.', "917888") + size = int(body.get("size", 0)) + if size < MIN_VOLUME: + raise _err(400, f"Volume size {size} is too small; minimum is {MIN_VOLUME}.", + "917534") + aggrs = body.get("aggregates") or [] + if len(aggrs) > 1: + # Several aggregates would create a FlexGroup: the adapter must not. + raise _err(400, "Specifying multiple aggregates creates a FlexGroup volume, " + "which is not supported for LUNs.", "917833") + known = {a["name"]: a for a in self.aggregates} + for a in aggrs: + if a.get("name") not in known: + raise _err(404, f'Aggregate "{a.get("name")}" does not exist.', "917927") + size = -(-size // 4096) * 4096 + + def create() -> None: + if any(v["name"] == name and v["svm"]["name"] == svm["name"] + for v in self.volumes.values()): + raise _err(409, f'Duplicate volume name "{name}".', "917535") + vid = self.add_flexvol(name, size, svm["name"], + int(((body.get("space") or {}).get("snapshot") or {}) + .get("reserve_percent", 5))) + vol = self.volumes[vid] + vol["guarantee"] = {"type": (body.get("guarantee") or {}).get("type", "volume")} + vol["snapshot_policy"] = {"name": (body.get("snapshot_policy") or {}) + .get("name", "default")} + vol["comment"] = body.get("comment", "") + if aggrs: + a = known[aggrs[0]["name"]] + vol["aggregates"] = [{"name": a["name"], "uuid": a["uuid"]}] + + return self._job(f"POST /api/storage/volumes/{name}", create) + + @route("PATCH", "/api/storage/volumes/" + _U) + def patch_volume(self, req, uuid): + self._auth(req) + if uuid not in self.volumes: + raise _not_found() + body = req.json() or {} + vol = self.volumes[uuid] + + def modify() -> None: + if "size" in body: + size = -(-int(body["size"]) // 4096) * 4096 + reserve = vol["space"]["snapshot"]["reserve_percent"] + biggest = max([lun["space"]["size"] for lun in self._luns_in(uuid)] or [0]) + if size < MIN_VOLUME or size * (100 - reserve) // 100 < biggest: + raise _err(400, "Cannot shrink the volume below the space used by its " + "LUNs or namespaces.", "917627") + vol["size"] = size + + return self._job(f"PATCH /api/storage/volumes/{uuid}", modify) + + def _luns_in(self, vol_uuid: str) -> list[dict[str, Any]]: + objs = list(self.luns.values()) + list(self.namespaces.values()) + return [o for o in objs if o["location"]["volume"]["uuid"] == vol_uuid] + + @route("DELETE", "/api/storage/volumes/" + _U) + def delete_volume(self, req, uuid): + self._auth(req) + if uuid not in self.volumes: + raise _not_found() + + force = req.q("force") == "true" + + def delete() -> None: + contained = self._luns_in(uuid) + if any(o["status"]["mapped"] for o in contained): + raise _err(400, "Volume contains mapped LUNs or namespaces.", "917536") + for o in contained: + self.luns.pop(o["uuid"], None) + self.namespaces.pop(o["uuid"], None) + vol = self.volumes.pop(uuid) + if not force: + self.recovery_queue[uuid] = vol + + return self._job(f"DELETE /api/storage/volumes/{uuid}", delete) + + # -- LUNs and namespaces ------------------------------------------------------ + + def _check_size(self, vol: dict[str, Any], size: int, block: int = 512) -> None: + if size < 4096 or size > MAX_LUN: + raise _err(400, f"Size {size} is out of range (4096..{MAX_LUN}).", "5374855") + if size % block: + raise _err(400, f"Size {size} is not a multiple of the block size {block}.", + "72089652") + if size > self._usable(vol): + raise _err(400, f'Size {size} exceeds the space available in volume ' + f'"{vol["name"]}".', "5374857") + + @route("GET", "/api/storage/luns") + def get_luns(self, req): + self._auth(req) + return self._collection(req, list(self.luns.values())) + + @route("GET", "/api/storage/luns/" + _U) + def get_lun(self, req, uuid): + self._auth(req) + if uuid not in self.luns: + raise _not_found() + return Reply(200, copy.deepcopy(self.luns[uuid])) + + @route("POST", "/api/storage/luns") + def post_lun(self, req): + self._auth(req) + body = req.json() or {} + svm = self._svm_of(body) + path = body.get("name", "") + vol = self._vol_of_path(path, svm["name"]) + if not body.get("os_type"): + raise _err(400, 'Field "os_type" is required.', "2") + + def run() -> Reply: + if any(o["name"] == path for o in list(self.luns.values()) + + list(self.namespaces.values())): + raise _err(409, f'A LUN or namespace already exists at path "{path}".', + "5374242") + size = int((body.get("space") or {}).get("size", 0)) + self._check_size(vol, size) + lid = self.add_lun(path, size, svm["name"]) + lun = self.luns[lid] + lun["os_type"] = body["os_type"] + requested = bool(((body.get("space") or {}).get("guarantee") or {}) + .get("requested")) + lun["space"]["guarantee"] = {"requested": requested, "reserved": requested} + return self._created(req, lun, f"/api/storage/luns/{lid}") + + return self._san(f"POST /api/storage/luns {path}", run) + + @route("PATCH", "/api/storage/luns/" + _U) + def patch_lun(self, req, uuid): + self._auth(req) + lun = self.luns.get(uuid) + if lun is None: + raise _not_found() + size = ((req.json() or {}).get("space") or {}).get("size") + + def run() -> Reply: + if size is not None: + new = int(size) + if new < lun["space"]["size"]: + raise _err(400, "Reducing the size of a LUN might cause permanent data " + "loss or corruption and is not supported by the LUN REST " + "API.", "5374923") + self._check_size(self.volumes[lun["location"]["volume"]["uuid"]], new) + lun["space"]["size"] = new + return Reply(200, {}) + + return self._san(f"PATCH /api/storage/luns/{uuid}", run) + + @route("DELETE", "/api/storage/luns/" + _U) + def delete_lun(self, req, uuid): + self._auth(req) + lun = self.luns.get(uuid) + if lun is None: + raise _not_found() + allow = req.q("allow_delete_while_mapped") == "true" + + def run() -> Reply: + if lun["status"]["mapped"] and not allow: + raise _err(400, "The LUN is mapped; unmap it or set " + "allow_delete_while_mapped.", "5374875") + self.lun_maps = [m for m in self.lun_maps if m["lun"]["uuid"] != uuid] + self.luns.pop(uuid, None) + return Reply(200, {}) + + return self._san(f"DELETE /api/storage/luns/{uuid}", run) + + @route("GET", "/api/storage/namespaces") + def get_namespaces(self, req): + self._auth(req) + return self._collection(req, list(self.namespaces.values())) + + @route("POST", "/api/storage/namespaces") + def post_namespace(self, req): + self._auth(req) + body = req.json() or {} + svm = self._svm_of(body) + path = body.get("name", "") + vol = self._vol_of_path(path, svm["name"]) + if not body.get("os_type"): + raise _err(400, 'Field "os_type" is required.', "2") + + def run() -> Reply: + if any(o["name"] == path for o in list(self.luns.values()) + + list(self.namespaces.values())): + raise _err(409, f'A LUN or namespace already exists at path "{path}".', + "72090019") + space = body.get("space") or {} + block = int(space.get("block_size", 4096)) + if block not in (512, 4096): + raise _err(400, "block_size must be 512 or 4096.", "72089653") + size = int(space.get("size", 0)) + self._check_size(vol, size, block) + nid = str(uuid_mod.uuid4()) + self.namespaces[nid] = { + "uuid": nid, "name": path, "svm": svm, "os_type": body["os_type"], + "space": {"size": size, "block_size": block}, + "location": {"namespace": path.rsplit("/", 1)[1], + "volume": {"name": vol["name"], "uuid": vol["uuid"]}}, + "status": {"mapped": False, "state": "online"}, + } + return self._created(req, self.namespaces[nid], f"/api/storage/namespaces/{nid}") + + return self._san(f"POST /api/storage/namespaces {path}", run) + + @route("PATCH", "/api/storage/namespaces/" + _U) + def patch_namespace(self, req, uuid): + self._auth(req) + ns = self.namespaces.get(uuid) + if ns is None: + raise _not_found() + size = ((req.json() or {}).get("space") or {}).get("size") + + def run() -> Reply: + if size is not None: + new = int(size) + if new < ns["space"]["size"]: + raise _err(400, "Shrinking a namespace is not supported.", "72090031") + self._check_size(self.volumes[ns["location"]["volume"]["uuid"]], new, + ns["space"]["block_size"]) + ns["space"]["size"] = new + return Reply(200, {}) + + return self._san(f"PATCH /api/storage/namespaces/{uuid}", run) + + @route("DELETE", "/api/storage/namespaces/" + _U) + def delete_namespace(self, req, uuid): + self._auth(req) + ns = self.namespaces.get(uuid) + if ns is None: + raise _not_found() + allow = req.q("allow_delete_while_mapped") == "true" + + def run() -> Reply: + if ns["status"]["mapped"] and not allow: + raise _err(400, "The namespace is mapped to a subsystem.", "72090024") + self.subsystem_maps = [m for m in self.subsystem_maps + if m["namespace"]["uuid"] != uuid] + self.namespaces.pop(uuid, None) + return Reply(200, {}) + + return self._san(f"DELETE /api/storage/namespaces/{uuid}", run) + + # -- igroups ------------------------------------------------------------------- + + def _check_initiator(self, group: dict[str, Any], name: str) -> None: + proto = group["protocol"] + is_iscsi = name.lower().startswith(("iqn.", "eui.", "naa.")) + is_fc = bool(_WWPN_RE.match(name)) + if not (is_iscsi or is_fc) or (proto == "iscsi" and not is_iscsi) or ( + proto == "fcp" and not is_fc): + raise _err(400, f'Initiator "{name}" is not valid for an igroup of protocol ' + f'"{proto}".', "1254324") + for g in self.igroups.values(): + if g is group or g["svm"]["name"] != group["svm"]["name"]: + continue + if name.lower() in (i["name"].lower() for i in g["initiators"]) and \ + g["os_type"] != group["os_type"]: + raise _err(400, f'Initiator "{name}" is in igroup "{g["name"]}" with a ' + "different os_type.", "1254323") + if name.lower() in (i["name"].lower() for i in group["initiators"]): + raise _err(409, f'Initiator "{name}" is already in igroup "{group["name"]}".', + "1254327") + + @route("GET", "/api/protocols/san/igroups") + def get_igroups(self, req): + self._auth(req) + return self._collection(req, list(self.igroups.values())) + + @route("POST", "/api/protocols/san/igroups") + def post_igroup(self, req): + self._auth(req) + body = req.json() or {} + svm = self._svm_of(body) + name = body.get("name", "") + if not _IGROUP_RE.match(name): + raise _err(400, f'Invalid igroup name "{name}".', "5374737") + if any(g["name"] == name and g["svm"]["name"] == svm["name"] + for g in self.igroups.values()): + raise _err(409, f'An igroup named "{name}" already exists.', "5374734") + proto = body.get("protocol", "mixed") + if proto not in ("iscsi", "fcp", "mixed"): + raise _err(400, f'Invalid protocol "{proto}".', "2") + gid = str(uuid_mod.uuid4()) + group = {"uuid": gid, "name": name, "svm": svm, "protocol": proto, + "os_type": body.get("os_type", "linux"), "initiators": []} + for i in body.get("initiators") or []: + self._check_initiator(group, i["name"]) + group["initiators"].append({"name": i["name"]}) + self.igroups[gid] = group + return self._created(req, group, f"/api/protocols/san/igroups/{gid}") + + @route("PATCH", "/api/protocols/san/igroups/" + _U) + def patch_igroup(self, req, uuid): + self._auth(req) + group = self.igroups.get(uuid) + if group is None: + raise _not_found() + proto = (req.json() or {}).get("protocol") + if proto is not None and proto != group["protocol"]: + # ONTAP spec, igroup.protocol: "The protocol of an initiator group + # cannot be changed after creation of the group." + raise _err(400, "The protocol of an initiator group cannot be changed after " + "creation of the group.", "5374746") + return Reply(200, {}) + + @route("POST", "/api/protocols/san/igroups/" + _U + "/initiators") + def post_igroup_initiators(self, req, uuid): + self._auth(req) + group = self.igroups.get(uuid) + if group is None: + raise _not_found() + body = req.json() or {} + records = body.get("records") or [body] + for r in records: + self._check_initiator(group, r["name"]) + for r in records: + group["initiators"].append({"name": r["name"]}) + return Reply(201, {}) + + @route("DELETE", "/api/protocols/san/igroups/" + _U) + def delete_igroup(self, req, uuid): + self._auth(req) + if uuid not in self.igroups: + raise _not_found() + if any(m["igroup"]["uuid"] == uuid for m in self.lun_maps) and \ + req.q("allow_delete_while_mapped") != "true": + raise _err(400, "The igroup is mapped to one or more LUNs.", "5374852") + del self.igroups[uuid] + return Reply(200, {}) + + # -- lun-maps ---------------------------------------------------------------------- + + @route("GET", "/api/protocols/san/lun-maps") + def get_lun_maps(self, req): + self._auth(req) + return self._collection(req, self.lun_maps, keys=("svm", "lun", "igroup")) + + def _find(self, table: dict[str, dict[str, Any]], ref: dict[str, Any], svm: str, + what: str) -> dict[str, Any]: + for obj in table.values(): + if obj["svm"]["name"] != svm: + continue + if ref.get("uuid") == obj["uuid"] or (ref.get("name") and + ref.get("name") == obj["name"]): + return obj + raise _err(404, f'{what} "{ref.get("name") or ref.get("uuid")}" does not exist.', "4") + + @route("POST", "/api/protocols/san/lun-maps") + def post_lun_map(self, req): + self._auth(req) + body = req.json() or {} + svm = self._svm_of(body) + lun = self._find(self.luns, body.get("lun") or {}, svm["name"], "LUN") + group = self._find(self.igroups, body.get("igroup") or {}, svm["name"], "igroup") + if any(m["lun"]["uuid"] == lun["uuid"] and m["igroup"]["uuid"] == group["uuid"] + for m in self.lun_maps): + raise _err(409, "The LUN is already mapped to this igroup.", "5374922") + used = {m["logical_unit_number"] for m in self.lun_maps + if m["igroup"]["uuid"] == group["uuid"]} + number = body.get("logical_unit_number") + if number is None: + number = next(n for n in range(4096) if n not in used) + elif number in used: + raise _err(409, f"LUN number {number} is in use in this igroup.", "5374921") + m = {"svm": svm, "lun": {"uuid": lun["uuid"], "name": lun["name"]}, + "igroup": {"uuid": group["uuid"], "name": group["name"]}, + "logical_unit_number": number} + self.lun_maps.append(m) + lun["status"]["mapped"] = True + return self._created(req, m, f"/api/protocols/san/lun-maps/{lun['uuid']}/" + f"{group['uuid']}") + + @route("DELETE", "/api/protocols/san/lun-maps/" + r"(?P[0-9a-fA-F-]{36})/" + + r"(?P[0-9a-fA-F-]{36})") + def delete_lun_map(self, req, lun, igroup): + self._auth(req) + before = len(self.lun_maps) + self.lun_maps = [m for m in self.lun_maps + if not (m["lun"]["uuid"] == lun and m["igroup"]["uuid"] == igroup)] + if len(self.lun_maps) == before: + raise _not_found() + if lun in self.luns: + self.luns[lun]["status"]["mapped"] = any( + m["lun"]["uuid"] == lun for m in self.lun_maps) + return Reply(200, {}) + + # -- NVMe subsystems ----------------------------------------------------------------- + + @route("GET", "/api/protocols/nvme/subsystems") + def get_subsystems(self, req): + self._auth(req) + return self._collection(req, list(self.subsystems.values())) + + @staticmethod + def _check_nqn(nqn: str) -> None: + if not nqn.startswith("nqn."): + raise _err(400, f'Invalid host NQN "{nqn}".', "72089705") + + @route("POST", "/api/protocols/nvme/subsystems") + def post_subsystem(self, req): + self._auth(req) + body = req.json() or {} + svm = self._svm_of(body) + name = body.get("name", "") + if not _SUBSYS_RE.match(name): + raise _err(400, f'Invalid subsystem name "{name}".', "72089771") + if not body.get("os_type"): + raise _err(400, 'Field "os_type" is required.', "2") + if any(s["name"] == name and s["svm"]["name"] == svm["name"] + for s in self.subsystems.values()): + raise _err(409, f'Subsystem "{name}" already exists.', "72090025") + hosts = [] + for h in body.get("hosts") or []: + self._check_nqn(h["nqn"]) + hosts.append({"nqn": h["nqn"]}) + sid = str(uuid_mod.uuid4()) + self.subsystems[sid] = {"uuid": sid, "name": name, "svm": svm, + "os_type": body["os_type"], "comment": body.get("comment", ""), + "hosts": hosts, + "target_nqn": f"nqn.1992-08.com.netapp:sn.{sid.replace('-', '')}" + f":subsystem.{name}"} + return self._created(req, self.subsystems[sid], f"/api/protocols/nvme/subsystems/{sid}") + + @route("POST", "/api/protocols/nvme/subsystems/" + _U + "/hosts") + def post_subsystem_hosts(self, req, uuid): + self._auth(req) + sub = self.subsystems.get(uuid) + if sub is None: + raise _not_found() + body = req.json() or {} + records = body.get("records") or [body] + have = {h["nqn"] for h in sub["hosts"]} + for r in records: + self._check_nqn(r["nqn"]) + if r["nqn"] in have: + raise _err(409, f'Host "{r["nqn"]}" is already in subsystem "{sub["name"]}".', + "72089708") + sub["hosts"].extend({"nqn": r["nqn"]} for r in records) + return Reply(201, {}) + + @route("DELETE", "/api/protocols/nvme/subsystems/" + _U + "/hosts/(?P[^/]+)") + def delete_subsystem_host(self, req, uuid, nqn): + self._auth(req) + sub = self.subsystems.get(uuid) + if sub is None: + raise _not_found() + nqn = unquote(nqn) + if nqn not in {h["nqn"] for h in sub["hosts"]}: + raise _not_found() + sub["hosts"] = [h for h in sub["hosts"] if h["nqn"] != nqn] + return Reply(200, {}) + + @route("DELETE", "/api/protocols/nvme/subsystems/" + _U) + def delete_subsystem(self, req, uuid): + self._auth(req) + sub = self.subsystems.get(uuid) + if sub is None: + raise _not_found() + if any(m["subsystem"]["uuid"] == uuid for m in self.subsystem_maps) and \ + req.q("allow_delete_while_mapped") != "true": + raise _err(400, "The subsystem has mapped namespaces.", "72090028") + if sub["hosts"] and req.q("allow_delete_with_hosts") != "true": + raise _err(400, "The subsystem has hosts; set allow_delete_with_hosts.", + "72090026") + for m in [m for m in self.subsystem_maps if m["subsystem"]["uuid"] == uuid]: + self.namespaces[m["namespace"]["uuid"]]["status"]["mapped"] = False + self.subsystem_maps = [m for m in self.subsystem_maps if m["subsystem"]["uuid"] != uuid] + del self.subsystems[uuid] + return Reply(200, {}) + + # -- subsystem-maps -------------------------------------------------------------------- + + @route("GET", "/api/protocols/nvme/subsystem-maps") + def get_subsystem_maps(self, req): + self._auth(req) + return self._collection(req, self.subsystem_maps, keys=("svm", "namespace", "subsystem")) + + @route("POST", "/api/protocols/nvme/subsystem-maps") + def post_subsystem_map(self, req): + self._auth(req) + body = req.json() or {} + svm = self._svm_of(body) + ns = self._find(self.namespaces, body.get("namespace") or {}, svm["name"], "Namespace") + sub = self._find(self.subsystems, body.get("subsystem") or {}, svm["name"], + "Subsystem") + if any(m["namespace"]["uuid"] == ns["uuid"] for m in self.subsystem_maps): + raise _err(400, "The namespace is already mapped to a subsystem. A namespace " + "can be mapped to only one subsystem.", "72089771") + used = {int(m["nsid"].rstrip("h"), 16) for m in self.subsystem_maps + if m["subsystem"]["uuid"] == sub["uuid"]} + nsid = next(n for n in range(1, 4096) if n not in used) + m = {"svm": svm, "namespace": {"uuid": ns["uuid"], "name": ns["name"]}, + "subsystem": {"uuid": sub["uuid"], "name": sub["name"]}, + "nsid": f"{nsid:08x}h"} + self.subsystem_maps.append(m) + ns["status"]["mapped"] = True + return self._created(req, m, f"/api/protocols/nvme/subsystem-maps/{sub['uuid']}/" + f"{ns['uuid']}") + + @route("DELETE", "/api/protocols/nvme/subsystem-maps/" + r"(?P[0-9a-fA-F-]{36})/" + + r"(?P[0-9a-fA-F-]{36})") + def delete_subsystem_map(self, req, sub, ns): + self._auth(req) + before = len(self.subsystem_maps) + self.subsystem_maps = [m for m in self.subsystem_maps + if not (m["subsystem"]["uuid"] == sub + and m["namespace"]["uuid"] == ns)] + if len(self.subsystem_maps) == before: + raise _not_found() + if ns in self.namespaces: + self.namespaces[ns]["status"]["mapped"] = False + return Reply(200, {}) + + def not_routed(self, req: Request) -> Reply: + return Reply(404, {"error": {"message": f"API not found: {req.method} {req.path}", + "code": "3"}}) + + +def contract_setup(**options: Any) -> ContractSetup: + fake = FakeNetAppOntap(username="admin", password="netapp1!", svm="svm1") + return ContractSetup( + fake=fake, + config=make_config("netapp-ontap", "https://ontap.fake.example", + **{"svm": "svm1", **options}), + credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("netapp1!")}), + bad_credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("wrong")}), + # Default protocol_family is scsi: the contract maps every volume to + # hosts of each listed protocol, so only SCSI initiators are listed. + # NVMe is covered by tests/unit/storage_arrays/test_netapp_ontap.py. + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:FF:4C:1A:2B"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_foreign_lun(name, size), + ) diff --git a/simplyblock_core/storage_arrays/testing/pure.py b/simplyblock_core/storage_arrays/testing/pure.py new file mode 100644 index 0000000000..bc6e417038 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/pure.py @@ -0,0 +1,290 @@ +"""Fake Pure FlashArray REST 2.x (the subset the adapter uses). + +Modelled on py-pure-client (``_common/apis/*_v_2_36.py``, ``models``): +connections of NVMe hosts carry ``nsid`` and no ``lun`` (REST 2.27+), SCSI +ones ``lun``; errors are HTTP 400 ``{"errors": [{"message", "context"}]}``. +""" +from __future__ import annotations + +import secrets +from typing import Any, ClassVar + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +_V = r"/api/(?P2\.\d+)" + + +def _err(msg: str, status: int = 400) -> FakeError: + return FakeError(status, {"errors": [{"message": msg, "context": ""}]}) + + +class FakePureFlashArray(FakeArray): + vendor = "pure-flasharray" + versions: ClassVar[list[str]] = ["1.19", "2.0", "2.4", "2.17", "2.21", "2.36", "2.40"] + + def __init__(self, api_token: str = "pure-token"): + super().__init__() + self.api_token = api_token + self.sessions: set[str] = set() + self.volumes: dict[str, dict[str, Any]] = {} # id -> volume + self.hosts: dict[str, dict[str, Any]] = {} # name -> host + self.connections: list[dict[str, Any]] = [] # {host, volume_id, lun|nsid} + + # -- fake controls ------------------------------------------------------ + + def expire_sessions(self) -> None: + self.sessions.clear() + + def set_credential(self, **fields: str) -> None: + self.api_token = fields["api_token"] + + def add_volume(self, name: str, size: int) -> str: + vid = f"{secrets.token_hex(4)}-{self.next_id()}" + self.volumes[vid] = {"id": vid, "name": name, "provisioned": size, + "serial": secrets.token_hex(12).upper(), "destroyed": False} + return vid + + # -- auth --------------------------------------------------------------- + + def _auth(self, req: Request) -> None: + if req.headers.get("x-auth-token") not in self.sessions: + raise _err("Authentication failed.", 401) + + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return [v["id"] for v in self.volumes.values() if not v.get("destroyed")] + + def lio_all_iqns(self) -> list[str]: + return sorted({str(i).lower() for h in self.hosts.values() for i in h.get("iqns", [])}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + host = self.hosts.get(host_id) + return [str(i).lower() for i in (host or {}).get("iqns", [])] + + @route("GET", "/api/api_version") + def api_version(self, req): + return Reply(200, {"version": self.versions}) + + @route("POST", _V + "/login") + def login(self, req, ver): + if req.headers.get("api-token") != self.api_token: + raise _err("Invalid API token.", 401) + token = secrets.token_hex(8) + self.sessions.add(token) + return Reply(200, {"items": [{"username": "sb"}]}, {"x-auth-token": token}) + + @route("POST", _V + "/logout") + def logout(self, req, ver): + self.sessions.discard(req.headers.get("x-auth-token")) + return Reply(200, {}) + + @route("GET", _V + "/arrays") + def arrays(self, req, ver): + self._auth(req) + return Reply(200, {"items": [{"id": "fake-array-id", "name": "fake-fa", + "os": "Purity//FA", "version": "6.5.0"}]}) + + @route("GET", _V + "/controllers") + def controllers(self, req, ver): + self._auth(req) + return Reply(200, {"items": [{"name": "CT0", "model": "FA-X70R4"}]}) + + # -- volumes ------------------------------------------------------------- + + def _select(self, req) -> list[dict[str, Any]]: + if req.q("ids"): + ids = req.q("ids").split(",") + missing = [i for i in ids if i not in self.volumes] + if missing: + raise _err("Volume does not exist.") + return [self.volumes[i] for i in ids] + names = (req.q("names") or "").split(",") + found = [v for v in self.volumes.values() if v["name"] in names] + if len(found) < len(names): + raise _err("Volume does not exist.") + return found + + @route("GET", _V + "/volumes") + def get_volumes(self, req, ver): + self._auth(req) + return Reply(200, {"items": self._select(req)}) + + @route("POST", _V + "/volumes") + def post_volumes(self, req, ver): + self._auth(req) + name = req.q("names") + body = req.json() + if any(v["name"] == name for v in self.volumes.values()): + raise _err("Volume already exists.") + size = int(body["provisioned"]) + if size % 512: + raise _err("Volume size must be a multiple of 512.") + vid = self.add_volume(name, size) + return Reply(200, {"items": [self.volumes[vid]]}) + + @route("PATCH", _V + "/volumes") + def patch_volumes(self, req, ver): + self._auth(req) + body = req.json() + vols = self._select(req) + for v in vols: + if "provisioned" in body: + size = int(body["provisioned"]) + if size < v["provisioned"] and not body.get("truncate"): + raise _err("Implicit truncation is not permitted.") + v["provisioned"] = size + if body.get("destroyed"): + if any(c["volume_id"] == v["id"] for c in self.connections): + raise _err("Volume has host connections.") + v["destroyed"] = True + return Reply(200, {"items": vols}) + + @route("DELETE", _V + "/volumes") + def delete_volumes(self, req, ver): + self._auth(req) + for v in self._select(req): + if not v["destroyed"]: + raise _err("Volume must be destroyed before eradication.") + del self.volumes[v["id"]] + return Reply(200, {}) + + # -- hosts ---------------------------------------------------------------- + + def _owner(self, key: str, value: str) -> str | None: + for h in self.hosts.values(): + if value.lower() in (x.lower() for x in h[key]): + return h["name"] + return None + + @route("GET", _V + "/hosts") + def get_hosts(self, req, ver): + self._auth(req) + name = req.q("names") + if name not in self.hosts: + raise _err("Host does not exist.") + return Reply(200, {"items": [self.hosts[name]]}) + + def _claim(self, host: str, body: dict[str, Any], prefix: str = "") -> None: + for key in ("iqns", "wwns", "nqns"): + for value in body.get(prefix + key, []): + owner = self._owner(key, value) + if owner and owner != host: + raise _err(f"{value} is already in use by host {owner}.") + if key == "wwns" and len(value.replace(":", "")) != 16: + raise _err("Invalid WWN.") + if value.lower() not in (x.lower() for x in self.hosts[host][key]): + self.hosts[host][key].append(value) + + @route("POST", _V + "/hosts") + def post_hosts(self, req, ver): + self._auth(req) + name = req.q("names") + if name in self.hosts: + raise _err("Host already exists.") + self.hosts[name] = {"name": name, "iqns": [], "wwns": [], "nqns": []} + try: + self._claim(name, req.json() or {}) + except FakeError: + del self.hosts[name] + raise + return Reply(200, {"items": [self.hosts[name]]}) + + @route("PATCH", _V + "/hosts") + def patch_hosts(self, req, ver): + self._auth(req) + name = req.q("names") + if name not in self.hosts: + raise _err("Host does not exist.") + self._claim(name, req.json() or {}, prefix="add_") + return Reply(200, {"items": [self.hosts[name]]}) + + @route("DELETE", _V + "/hosts") + def delete_hosts(self, req, ver): + self._auth(req) + name = req.q("names") + if name not in self.hosts: + raise _err("Host does not exist.") + if any(c["host"] == name for c in self.connections): + raise _err("Host has volume connections.") + del self.hosts[name] + return Reply(200, {}) + + # -- connections ------------------------------------------------------------ + + def _conn_item(self, c): + v = self.volumes[c["volume_id"]] + item = {"host": {"name": c["host"]}, "host_group": None, + "volume": {"id": v["id"], "name": v["name"]}, "lun": None, "nsid": None} + item["nsid" if "nsid" in c else "lun"] = c.get("nsid", c.get("lun")) + return item + + @route("GET", _V + "/connections") + def get_connections(self, req, ver): + self._auth(req) + vid, host = req.q("volume_ids"), req.q("host_names") + if vid and vid not in self.volumes: + raise _err("Volume does not exist.") + items = [self._conn_item(c) for c in self.connections + if (not vid or c["volume_id"] == vid) and (not host or c["host"] == host)] + return Reply(200, {"items": items}) + + @route("POST", _V + "/connections") + def post_connections(self, req, ver): + self._auth(req) + vid, host = req.q("volume_ids"), req.q("host_names") + if vid not in self.volumes or self.volumes[vid]["destroyed"]: + raise _err("Volume does not exist.") + if host not in self.hosts: + raise _err("Host does not exist.") + if any(c["volume_id"] == vid and c["host"] == host for c in self.connections): + raise _err("Connection already exists.") + key = "nsid" if self.hosts[host]["nqns"] else "lun" + used = {c.get(key) for c in self.connections if c["host"] == host} + c = {"host": host, "volume_id": vid, + key: next(n for n in range(1, 4096) if n not in used)} + self.connections.append(c) + return Reply(200, {"items": [self._conn_item(c)]}) + + @route("DELETE", _V + "/connections") + def delete_connections(self, req, ver): + self._auth(req) + vid, host = req.q("volume_ids"), req.q("host_names") + before = len(self.connections) + self.connections = [c for c in self.connections + if not (c["volume_id"] == vid and c["host"] == host)] + if len(self.connections) == before: + raise _err("Connection does not exist.") + return Reply(200, {}) + + +def contract_setup() -> ContractSetup: + fake = FakePureFlashArray(api_token="pure-token") + return ContractSetup( + fake=fake, + config=make_config("pure-flasharray", "https://pure.fake.example"), + credential=Credential({"api_token": SecretStr("pure-token")}), + bad_credential=Credential({"api_token": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + Protocol.NVME_TCP: ( + (Initiator(protocol=Protocol.NVME_TCP, + address="nqn.2014-08.org.nvmexpress:uuid:host-a"),), + (Initiator(protocol=Protocol.NVME_TCP, + address="nqn.2014-08.org.nvmexpress:uuid:host-b"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_volume(name, size), + ) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 7e09b48725..8c6b818c08 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -36,7 +36,9 @@ cluster_ops, constants, distr_controller, + floors, jm_raid, + multi_attach, scripts, storage_network, utils, @@ -49,6 +51,7 @@ ) from simplyblock_core.controllers import ( activation_gate, + array_controller, device_controller, device_events, health_controller, @@ -1088,6 +1091,16 @@ def _collect_attached_ips(ctrlr_list): return attached +def _serving_node_id(device) -> str: + """The node exporting ``device``: ``serving_node()`` for an NVMeDevice + (the owner unless a multi-attach takeover moved its LUN), ``node_id`` for + any other device record (JM devices).""" + serving = getattr(device, "serving_node", None) + if callable(serving) and multi_attach.is_multi_attach(device): + return serving() + return device.node_id + + def connect_device(name: str, device: NVMeDevice, node: StorageNode, attach_timeout: float | None = None): """Connect snode to device @@ -1107,6 +1120,11 @@ def connect_device(name: str, device: NVMeDevice, node: StorageNode, attach_time expected_ips = [ip.strip() for ip in (device.nvmf_ip or "").split(",") if ip.strip()] is_multipath = bool(device.nvmf_multipath) and len(expected_ips) >= 2 + # Multi-attach: the controller can hold a dead path of the node that + # served the LUN before a takeover while its bdev lives on, so the + # controller is always inspected for the serving node's paths. + if multi_attach.is_multi_attach(device): + is_multipath = True rpc_client = node.rpc_client() # Fast path: bdev already present. Only safe for single-path devices — for @@ -1170,7 +1188,14 @@ def _connect_device_attach(name, device, node: StorageNode, rpc_client, attach_r # the budget, raise so the calling task suspends and retries. _TRANSIENT_STATES = ("failed", "resetting", "deleting", "reconnect_is_delayed") states: list[str] = [] + # Multi-attach: a path to the node that served the LUN before a + # takeover stays "resetting"/"reconnect_is_delayed" for the whole + # hold (MULTI_ATTACH_HOLD_SEC); waiting for it to settle would keep + # the new path from being added, which is what ends the hold. for _attempt in range(5): + if multi_attach.is_multi_attach(device): + logger.info(f"Controller found: {name}, multi-attach, not waiting on path states") + break if not ret: # The module destructed the controller on its own; the # fresh-attach path below takes over. @@ -1188,7 +1213,9 @@ def _connect_device_attach(name, device, node: StorageNode, rpc_client, attach_r raise RuntimeError(f"Controller: {name}, status is {states}") db_ctrl = DBController() - target_node = db_ctrl.get_storage_node_by_id(device.node_id) + # The node exporting the device: its owner, or the node a multi-attach + # takeover moved its LUN to. + target_node = db_ctrl.get_storage_node_by_id(_serving_node_id(device)) if target_node is not None and target_node.active_rdma: tr_type = "RDMA" elif target_node is not None and target_node.active_tcp: @@ -1205,6 +1232,11 @@ def _connect_device_attach(name, device, node: StorageNode, rpc_client, attach_r # ``"failover"`` (active-passive) and remote alceml/jm controllers # would carry all IO on a single path. attach_mode = "multipath" if device.nvmf_multipath else False + if multi_attach.is_multi_attach(device): + # A multi-attach device's controller must accept a further path (the + # LUN's export on a takeover node), which a controller attached with + # multipath disabled refuses. + attach_mode = "multipath" final = rpc_client.bdev_nvme_controller_list(name) if not final: @@ -1220,7 +1252,8 @@ def _connect_device_attach(name, device, node: StorageNode, rpc_client, attach_r try: resp = attach_rpc_client.bdev_nvme_attach_controller( name, device.nvmf_nqn, ip, device.nvmf_port, tr_type, - multipath=attach_mode) + multipath=attach_mode, + **multi_attach.attach_controller_kwargs(device)) if resp: dial_backoff.record_success(ip) else: @@ -1231,7 +1264,12 @@ def _connect_device_attach(name, device, node: StorageNode, rpc_client, attach_r dial_backoff.record_failure(ip) logger.warning(f"Failed to attach controller {name} via {ip}: {e}") - if device.nvmf_multipath and bdev_name: + if multi_attach.is_multi_attach(device) and bdev_name: + # One node serves the LUN at a time; its paths are the only + # valid ones (a takeover adds another node's, then drops the + # old). + rpc_client.bdev_nvme_set_multipath_policy(bdev_name, "active_passive") + elif device.nvmf_multipath and bdev_name: rpc_client.bdev_nvme_set_multipath_policy(bdev_name, "active_active") if not bdev_name: @@ -1275,7 +1313,8 @@ def _connect_device_attach(name, device, node: StorageNode, rpc_client, attach_r try: if attach_rpc_client.bdev_nvme_attach_controller( name, device.nvmf_nqn, ip, device.nvmf_port, tr_type, - multipath=attach_mode): + multipath=attach_mode, + **multi_attach.attach_controller_kwargs(device)): dial_backoff.record_success(ip) else: dial_backoff.record_failure(ip) @@ -1318,7 +1357,10 @@ def repair_multipath_controller(name: str, device, node: StorageNode): (i.e. the source NVMeDevice / JMDevice on the target node — NOT a ``RemoteJMDevice``, which strips those fields). """ - if not getattr(device, 'nvmf_multipath', False): + # Multi-attach: a single-path controller too can lack the path to the + # node a takeover moved the LUN to. + ma = multi_attach.is_multi_attach(device) + if not getattr(device, 'nvmf_multipath', False) and not ma: return True nvmf_ip = getattr(device, 'nvmf_ip', None) @@ -1332,7 +1374,7 @@ def repair_multipath_controller(name: str, device, node: StorageNode): return False expected_ips = {ip.strip() for ip in nvmf_ip.split(",") if ip.strip()} - if len(expected_ips) < 2: + if len(expected_ips) < 2 and not ma: return True # not actually multipath rpc_client = node.rpc_client() @@ -1341,7 +1383,7 @@ def repair_multipath_controller(name: str, device, node: StorageNode): return True # controller gone, connect_device will handle full reconnect db_ctrl = DBController() - target_node = db_ctrl.get_storage_node_by_id(device.node_id) + target_node = db_ctrl.get_storage_node_by_id(_serving_node_id(device)) if target_node is None: return False if target_node.active_rdma: @@ -1411,7 +1453,8 @@ def _repair_multipath_controller_locked(node, name, device, rpc_client, ret, # the 2026-08-20 soak, all of them "-5 Input/output error". if not rpc_client.bdev_nvme_attach_controller( name, device.nvmf_nqn, ip, device.nvmf_port, - tr_type, multipath="multipath"): + tr_type, multipath="multipath", + **multi_attach.attach_controller_kwargs(device)): dial_backoff.record_failure(ip) logger.warning( "Re-attach of path %s on controller %s was rejected by the " @@ -1445,7 +1488,8 @@ def cluster_growth_ratios(db_controller, cluster_id): """Growth of the data devices relative to their placement weight, as ``(uniform, min_ratio, max_ratio)``. Journal, new, and failed devices are ignored. A ratio above 1 means the device grew after it was placed (see - device_controller.resize_device).""" + device_controller.grow_cluster_devices, which grows all of them by the + same factor).""" ratios = [] for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id): if node.is_secondary_node: @@ -1471,8 +1515,8 @@ def scaled_placement_size(db_controller, cluster_id, size): a common factor f, their weights still stand for their original sizes, so a new device is weighted as size / f; weighting it by its full size would give it f times its share of the data. Returns 0 (weight = size) when the - cluster has not grown, or has grown unevenly (then the next - grow-devices run sets every weight to its size anyway).""" + cluster has not grown. Growth is uniform by construction; should the + ratios ever disagree, the device keeps its full size as weight.""" uniform, lo, hi = cluster_growth_ratios(db_controller, cluster_id) factor = (lo + hi) / 2 if not uniform or factor <= 1.0 + GROWTH_UNIFORM_TOLERANCE or not size: @@ -1970,6 +2014,8 @@ def _create_jm_stack_on_device(rpc_client, nvme, snode: StorageNode, after_resta 'alceml_name': alceml_name, 'nvme_bdev': nvme.nvme_bdev, "serial_number": nvme.serial_number, + 'aio_block_size': nvme.aio_block_size, + 'lblk_io_engine': nvme.lblk_io_engine, "device_data_dict": nvme.to_dict(), 'jm_bdev': jm_bdev, 'testing_bdev': test_name, @@ -2015,6 +2061,7 @@ def _create_storage_device_stack(rpc_client, nvme, snode: StorageNode, after_res cache_size=cache_size, cache_eviction_threshold=cache_eviction_threshold, force_4k_atomic=getattr(cluster, 'atomic_4k', False), + **multi_attach.alceml_create_kwargs(nvme, snode, cluster), ) if not ret: @@ -2034,9 +2081,12 @@ def _create_storage_device_stack(rpc_client, nvme, snode: StorageNode, after_res logger.error(f"Failed to read back pt bdev: {pt_name}") return None pt_spdk_uuid = pt_bdev_info["aliases"][0] - subsystem_nqn = snode.subsystem + ":dev:" + alceml_id + # Multi-attach: host-independent NQN and this node's cntlid range, so the + # LUN's export on another node can join the same controller as a path. + subsystem_nqn = multi_attach.device_subsystem_nqn(snode, nvme, cluster) logger.info("creating subsystem %s", subsystem_nqn) - ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', alceml_id) + ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', alceml_id, + **multi_attach.subsystem_create_kwargs(snode, nvme, db_controller)) ip_list = [] for iface in snode.data_nics: if iface.ip4_address: @@ -2045,7 +2095,8 @@ def _create_storage_device_stack(rpc_client, nvme, snode: StorageNode, after_res ip_list.append(iface.ip4_address) logger.info(f"add {pt_name} to subsystem") - ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name, alceml_id) + ns_extra = {"nguid": multi_attach.namespace_nguid(nvme)} if multi_attach.is_multi_attach(nvme) else {} + ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name, multi_attach.namespace_uuid(nvme), **ns_extra) if not ret: logger.error(f"Failed to add: {pt_name} to the subsystem: {subsystem_nqn}") return None @@ -2288,6 +2339,19 @@ def _prepare_cluster_devices_on_restart(snode: StorageNode, clear_data=False): logger.debug(f"Device is skipped: {nvme.get_id()}, status: {nvme.status}") continue + if multi_attach.not_servable_here(nvme, snode.get_id()): + # Multi-attach: another node took the LUN over and serves it. This + # node keeps the record and reaches the LUN through its loopback + # controller to the serving node; opening a second alceml on the + # LUN here would make two writers. No failback in this phase. + # Or this node is still fenced from the LUN (a takeover after its + # failure did not complete): the LUN cannot be opened until an + # operator unfences it. + logger.info(f"Device {nvme.get_id()} is not served by this node (serving node " + f"{nvme.serving_node()}, fenced: {multi_attach.fenced_from(nvme, snode.get_id())}); " + f"not building its stack here") + continue + jobs.append((nvme, _create_storage_device_stack, (rpc_client, nvme, snode, not clear_data,))) # Every device that reaches this point was found on the node and is @@ -2500,14 +2564,31 @@ def _connect_to_remote_devs( devices_to_connect = [] connect_threads = [] nodes = db_controller.get_storage_nodes_by_cluster_id(this_node.cluster_id) + # Multi-attach clusters: the node's own array-LUN devices are reached by + # its distribs through a loopback controller (remote_alceml_n1) like + # every peer's, so they are connected here too. Empty on other clusters. + own_loopback = multi_attach.own_multi_attach_devices(this_node, allowed_dev_statuses) + # Multi-attach clusters: a down owner's LUNs that another node took over + # are reachable through that node. Empty on other clusters. + nodes_by_id = {n.get_id(): n for n in nodes} + taken_over = {n.get_id(): multi_attach.served_by_online_node(n, nodes_by_id) + for n in nodes if n.status not in allowed_node_statuses} # connect to remote devs for node_index, node in enumerate(nodes): - if node.get_id() == this_node.get_id() or node.status not in allowed_node_statuses: - continue + if node.get_id() == this_node.get_id(): + if not own_loopback: + continue + node_devices = own_loopback + elif node.status not in allowed_node_statuses: + node_devices = taken_over.get(node.get_id()) or [] + if not node_devices: + continue + else: + node_devices = node.nvme_devices if only_node_id and node.get_id() != only_node_id: continue logger.info(f"Connecting to node {node.get_id()}") - for index, dev in enumerate(node.nvme_devices): + for index, dev in enumerate(node_devices): if dev.status not in allowed_dev_statuses: logger.debug(f"Device is not online: {dev.get_id()}, status: {dev.status}") @@ -2613,8 +2694,16 @@ def _find_remote_bdev(dev): # to per-device probes. _sweep_bdev_names = _fetch_bdev_name_set(rpc_client) for node in nodes: - if node.get_id() == this_node.get_id() or node.status not in allowed_node_statuses: - continue + if node.get_id() == this_node.get_id(): + if not own_loopback: + continue + node_devices = own_loopback + elif node.status not in allowed_node_statuses: + node_devices = taken_over.get(node.get_id()) or [] + if not node_devices: + continue + else: + node_devices = node.nvme_devices if only_node_id and node.get_id() != only_node_id: # DELTA mode: records for every other node were carried over # untouched above — probing their bdevs anyway made each delta @@ -2623,7 +2712,7 @@ def _find_remote_bdev(dev): # ~25 min, 2026-07-13 validation run). Verify only the delta # owner's devices. continue - for dev in node.nvme_devices: + for dev in node_devices: if dev.get_id() in remote_device_ids: continue if dev.status not in allowed_dev_statuses: @@ -2764,6 +2853,12 @@ def _verify_online_device_coverage(snode: StorageNode, repair: bool = True): if not dev.alceml_bdev: continue expected[f"remote_{dev.alceml_bdev}n1"] = dev + # Multi-attach clusters: the node's own array LUNs are mapped through its + # loopback controller, so they need one too (empty on other clusters). + for dev in multi_attach.own_multi_attach_devices( + snode, (NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, + NVMeDevice.STATUS_CANNOT_ALLOCATE)): + expected[f"remote_{dev.alceml_bdev}n1"] = dev if fd_recovery and excluded_fd_devs: # Never a silent relaxation — degraded recovery must be visible in logs. @@ -2834,12 +2929,24 @@ def sync_remote_devices_from_spdk(this_node: StorageNode): # device (O(cluster devices) per call). None -> per-device fallback. _sweep_bdev_names = _fetch_bdev_name_set(rpc_client) - for peer in db_controller.get_storage_nodes_by_cluster_id(fresh_node.cluster_id): + peers = db_controller.get_storage_nodes_by_cluster_id(fresh_node.cluster_id) + peers_by_id = {p.get_id(): p for p in peers} + for peer in peers: if peer.get_id() == fresh_node.get_id(): - continue - if peer.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_DOWN, StorageNode.STATUS_RESTARTING]: - continue - for dev in peer.nvme_devices: + # Multi-attach clusters: own array LUNs (loopback controllers). + peer_devices = multi_attach.own_multi_attach_devices( + fresh_node, (NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, + NVMeDevice.STATUS_CANNOT_ALLOCATE)) + if not peer_devices: + continue + elif peer.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_DOWN, StorageNode.STATUS_RESTARTING]: + # Multi-attach: its LUNs another node took over (empty otherwise). + peer_devices = multi_attach.served_by_online_node(peer, peers_by_id) + if not peer_devices: + continue + else: + peer_devices = peer.nvme_devices + for dev in peer_devices: if dev.status not in [ NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, @@ -2914,13 +3021,25 @@ def reconnect_dropped_remote_devs(this_node: StorageNode): changed = False all_ok = True - for peer in db_controller.get_storage_nodes_by_cluster_id(fresh_node.cluster_id): + peers = db_controller.get_storage_nodes_by_cluster_id(fresh_node.cluster_id) + peers_by_id = {p.get_id(): p for p in peers} + for peer in peers: if peer.get_id() == fresh_node.get_id(): - continue - if peer.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_DOWN, - StorageNode.STATUS_UNREACHABLE]: - continue - for dev in peer.nvme_devices: + # Multi-attach clusters: own array LUNs (loopback controllers). + peer_devices = multi_attach.own_multi_attach_devices( + fresh_node, (NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, + NVMeDevice.STATUS_CANNOT_ALLOCATE)) + if not peer_devices: + continue + elif peer.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_DOWN, + StorageNode.STATUS_UNREACHABLE]: + # Multi-attach: its LUNs another node took over (empty otherwise). + peer_devices = multi_attach.served_by_online_node(peer, peers_by_id) + if not peer_devices: + continue + else: + peer_devices = peer.nvme_devices + for dev in peer_devices: if dev.get_id() in known_ids: continue if dev.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, @@ -4283,8 +4402,27 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, minimum_sys_memory = int(utils.parse_size(spdk_sys_mem)) else: minimum_sys_memory = int(node_config.get("sys_memory") or 0) - if node_config.get("lblk_devices"): - minimum_sys_memory = max(minimum_sys_memory, utils.node_config_min_sys_memory(node_config)) + if node_config.get("lblk_devices") or node_config.get("lblk_array_only"): + # The LUNs the array plugin creates below count as well. + lun_capacity = 0 + if array_controller.provisions_luns(cluster): + lun_capacity = int(cluster.array_lun_size or 0) + if array_controller.journal_on_lun(cluster): + lun_capacity += array_controller.journal_lun_size(cluster) + minimum_sys_memory = max(minimum_sys_memory, + utils.node_config_min_sys_memory(node_config, lun_capacity)) + if multi_attach.warm_standby_cluster(cluster): + # The node will also hold the page map of the LUNs it is a + # warm standby of: FTT of them, each the size of its own + # (LUN sizes are uniform; growth is uniform-only). Sized + # here, the standbys open right away instead of waiting + # for the node's first restart (2026-11-20 bed: every + # standby refused for budget after add-node). + own = sum(int(e.get("size") or 0) + for e in node_config.get("lblk_devices") or []) + lun_capacity + ftt = max(1, int(getattr(cluster, "max_fault_tolerance", 1) or 1)) + minimum_sys_memory = max(minimum_sys_memory, utils.node_config_min_sys_memory( + node_config, lun_capacity + ftt * own)) else: ssd_pcis = node_config.get("ssd_pcis") or [] capacity = utils.nvme_capacity_from_node_info(node_info.get("nvme_devices"), ssd_pcis) @@ -4295,6 +4433,10 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, "devices visible (unbind them from SPDK) or pass spdk_sys_mem.", ssd_pcis, node_addr) return False minimum_sys_memory = max(minimum_sys_memory, utils.sys_memory_for_capacity(capacity)) + # Warm-standby multi-attach clusters: the lblk branch above sizes for + # FTT standby LUNs of the node's own size; from the first restart on + # the actual set counts (_warm_standby_capacity), and + # lun_takeover.ensure_warm_standbys never opens beyond the budget. max_lvol = node_config.get("max_lvol") # Clamp rather than fail: a config file generated before the cap was # enforced (or hand-edited) can carry a larger value, and refusing the @@ -4328,6 +4470,42 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, lblk_configured = node_config.get("lblk_devices") or [] lblk_mode = cluster.device_mode == constants.DEVICE_MODE_LBLK + # Multi-attach cluster with a registered array and a LUN size: the + # node's data LUN is created and presented here, then added like any + # configured lblk device. Its journal is either a local disk or + # partition the host's own config names (array journal "local"), or + # a journal LUN created here first (array journal "lun"; the host + # config names no device). No-op on every other cluster. + array_host_id = "" + if lblk_mode and array_controller.provisions_luns(cluster): + journal_lun = array_controller.journal_on_lun(cluster) + local = [e.get("name") for e in lblk_configured if not e.get("array_lun_id")] + if journal_lun and local: + logger.error( + "This cluster keeps each node's journal on a LUN of its array, but the " + f"node config names local devices {local}; run " + "'sn configure --lblk --lblk-array-only' first.") + return False + if not journal_lun and not local: + logger.error( + "This cluster keeps each node's journal on a local disk or partition, but " + "the node config names none; run 'sn configure --lblk --lblk-journal-only " + "--blk-names ' first, or register the array " + "with the journal on a LUN.") + return False + try: + if journal_lun: + journal_entry, array_host_id = array_controller.provision_node_lun( + cluster, snode_api, node_info, node_config, lblk_configured, + node_ip=node_addr.split(":")[0], journal=True) + lblk_configured = array_controller.add_lun_to_config(lblk_configured, journal_entry) + lun_entry, array_host_id = array_controller.provision_node_lun( + cluster, snode_api, node_info, node_config, lblk_configured, + node_ip=node_addr.split(":")[0]) + except array_controller.ArrayLifecycleError as e: + logger.error(f"Could not provision the node's array LUN: {e}") + return False + lblk_configured = array_controller.add_lun_to_config(lblk_configured, lun_entry) if lblk_mode: if not lblk_configured: logger.error( @@ -4579,7 +4757,8 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, logger.error(f"Failed to list block devices on {node_addr}: {err}") return False lblk_resolved, missing = utils.resolve_lblk_entries( - lblk_configured, blk_inventory) + lblk_configured, blk_inventory, + name_fallback=not multi_attach.is_multi_attach(cluster)) if missing: logger.error( f"Configured block device(s) not found on host: " @@ -4760,6 +4939,8 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, # lblk mode: persist the configured selection (with serial identity) # so restart can re-resolve devices without the host config file. snode.lblk_devices = lblk_configured if lblk_mode else [] + snode.array_host_id = array_host_id + snode.array_journal_lun_id = array_controller.journal_lun_id(lblk_configured) if lblk_mode else "" snode.hostname = hostname snode.k8s_node_name = node_info['hostname'] snode.host_nqn = subsystem_nqn @@ -4885,6 +5066,10 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, return False snode.write_to_db(kv_store) + if multi_attach.is_multi_attach(cluster): + # Stable controller-ID block of this node for its multi-attach + # device subsystems (smallest free index of the cluster). + multi_attach.ensure_cntlid_block(snode, db_controller) try: rpc_client.nvmf_set_max_subsystems(constants.NVMF_MAX_SUBSYSTEMS) @@ -4998,6 +5183,11 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, # discover devices if lblk_mode: nvme_devs = utils.addAioDevices(rpc_client, snode, lblk_resolved) + # Multi-attach clusters only: flag the array LUNs (no-op otherwise). + multi_attach.mark_lun_devices(cluster, snode, nvme_devs, lblk_resolved) + if multi_attach.is_multi_attach(cluster): + # Another node's LUN is never a device of this node. + nvme_devs = drop_foreign_luns(snode, nvme_devs) else: if not snode.ssd_pcie: node_info, _ = snode_api.info() @@ -5247,6 +5437,9 @@ def delete_storage_node(node_id, force=False): if snode.pod_network: # The Service held the node's address; with the record gone nothing names it. _delete_spdk_service(snode, snode.rpc_port, snode.cluster_id) + # Multi-attach cluster with an array: detach the node's LUNs from every + # host and other LUNs from its host (nothing is deleted; no-op otherwise). + array_controller.release_node_luns(snode) snode.remove(db_controller.kv_store) for node in db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id): @@ -8757,6 +8950,21 @@ def _finalize_node_removal(removed_node: StorageNode): logger.info("done") +def _warm_standby_capacity(db_controller, snode) -> int: + """Raw capacity of the LUNs ``snode`` keeps warm standbys of + (multi_attach.warm_standby_capacity); 0 when unknown or on any cluster + without Cluster.multi_attach_warm_standby.""" + try: + cluster = db_controller.get_cluster_by_id(snode.cluster_id) + if not multi_attach.is_multi_attach(cluster): + return 0 + return multi_attach.warm_standby_capacity( + cluster, snode.get_id(), db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)) + except Exception as e: # noqa: BLE001 + logger.warning("Warm standby capacity of %s unknown: %s", snode.get_id(), e) + return 0 + + def restart_storage_node( node_id, max_lvol=0, max_snap=0, max_prov=0, spdk_image=None, set_spdk_debug=None, @@ -9085,6 +9293,37 @@ def _new_restart_claim_token(): return f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:8]}" +def _foreign_lun_keys(snode): + """LUN identities of the other nodes' multi-attach devices, on a + multi-attach cluster; an empty set otherwise (no DB read).""" + db_controller = DBController() + try: + cluster = db_controller.get_cluster_by_id(snode.cluster_id) + except KeyError: + return set() + if not multi_attach.is_multi_attach(cluster): + return set() + return multi_attach.foreign_lun_keys( + db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id), snode.get_id()) + + +def drop_foreign_luns(snode, nvme_devs): + """``nvme_devs`` without the multi-attach LUNs of other nodes (add-node + discovery: they must never become devices of ``snode``, which would + format them). Unchanged on any other cluster.""" + foreign = _foreign_lun_keys(snode) + if not foreign: + return nvme_devs + kept = [] + for dev in nvme_devs: + if multi_attach.is_foreign_lun(dev, foreign): + logger.warning(f"Block device {dev.device_path or dev.device_name} (serial " + f"{dev.serial_number}) is another node's multi-attach LUN; skipped") + continue + kept.append(dev) + return kept + + def _reconcile_discovered_devices(snode, nvme_devs): """Match the devices just discovered on the host (`nvme_devs`, from addNvmeDevices/addAioDevices) against this node's known NVMeDevice @@ -9109,6 +9348,10 @@ def _reconcile_discovered_devices(snode, nvme_devs): known_devices_sn = [] admin_removed_sn = set() devices_sn_dict = {d.serial_number: d for d in nvme_devs} + # Multi-attach clusters: every LUN is presented to every node. A LUN of + # another node's device is never a new device here (it would be + # formatted). Empty on any other cluster. + foreign_luns = _foreign_lun_keys(snode) for db_dev in snode.nvme_devices: if db_dev.admin_removed: admin_removed_sn.add(db_dev.serial_number) @@ -9130,6 +9373,14 @@ def _reconcile_discovered_devices(snode, nvme_devs): db_dev.bdev_type = "aio" db_dev.device_path = found_dev.device_path db_dev.by_id_path = found_dev.by_id_path + # Restore a block size a control-plane service that + # predates the field dropped from the record (found_dev + # got it from the node's lblk_devices entry). + if found_dev.aio_block_size and not db_dev.aio_block_size: + db_dev.aio_block_size = found_dev.aio_block_size + # The I/O engine found_dev was created with (the recorded + # one, or uring for a device-mapper node). + db_dev.lblk_io_engine = found_dev.lblk_io_engine else: db_dev.nvme_controller = found_dev.nvme_controller db_dev.pcie_address = found_dev.pcie_address @@ -9138,6 +9389,13 @@ def _reconcile_discovered_devices(snode, nvme_devs): db_dev.md_supported = found_dev.md_supported active_devices.append(db_dev) + elif multi_attach.not_servable_here(db_dev, snode.get_id()): + # Multi-attach: the LUN is served by the node that took it over, + # or this node is fenced from it at the array; not seeing it does + # not mean it is gone. The record stays as it is. + logger.info(f"Device not found: {db_dev.get_id()}, served by node " + f"{db_dev.serving_node()}; record kept") + active_devices.append(db_dev) else: logger.info(f"Device not found: {db_dev.get_id()}") if db_dev.status == NVMeDevice.STATUS_NEW: @@ -9156,6 +9414,11 @@ def _reconcile_discovered_devices(snode, nvme_devs): logger.info(f"JM device found: {snode.jm_device.get_id()}") snode.jm_device.nvme_bdev = dev.nvme_bdev + elif dev.serial_number not in known_devices_sn and multi_attach.is_foreign_lun(dev, foreign_luns): + logger.warning(f"Block device {dev.device_path or dev.device_name} (serial " + f"{dev.serial_number}) is another node's multi-attach LUN; " + f"not added (never formatted)") + elif dev.serial_number not in known_devices_sn: if dev.serial_number in admin_removed_sn: logger.warning( @@ -9493,6 +9756,10 @@ def _restart_storage_node_impl( # The SPDK container's memory limit: recomputed from the node's devices # (the stored figure may come from an older formula), else the stored one. raw_capacity = sum(max(int(d.size or 0), 0) for d in (snode.nvme_devices or [])) + # Warm-standby multi-attach clusters: the node also holds the page map of + # every LUN it keeps a warm standby of (lun_takeover.ensure_warm_standbys), + # so those LUNs count as its own capacity. 0 on any other cluster. + raw_capacity += _warm_standby_capacity(db_controller, snode) minimum_sys_memory = (utils.sys_memory_for_capacity(raw_capacity) if raw_capacity else snode.minimum_sys_memory or 0) snode.spdk_mem = minimum_hp_memory @@ -9761,7 +10028,8 @@ def _restart_storage_node_impl( logger.error(f"Failed to list block devices: {blk_err}") return False lblk_resolved, lblk_missing = utils.resolve_lblk_entries( - snode.lblk_devices, blk_inventory) + snode.lblk_devices, blk_inventory, + name_fallback=not multi_attach.is_multi_attach(cluster)) for entry in lblk_missing: logger.warning(f"Configured block device {entry.get('path') or entry.get('name')} " f"(serial {entry.get('serial')}) not found on host") @@ -9769,6 +10037,9 @@ def _restart_storage_node_impl( if not nvme_devs: logger.error("No eligible block devices were found!") return False + # Multi-attach clusters only: a device first seen on this restart + # (new host) is an array LUN too. Existing records keep their flags. + multi_attach.mark_lun_devices(cluster, snode, nvme_devs, lblk_resolved) else: if not snode.ssd_pcie: ssds = node_info['spdk_pcie_list'] @@ -9872,6 +10143,10 @@ def _jm_reconcile(): snode = db_controller.get_storage_node_by_id(snode.get_id()) for db_dev in snode.nvme_devices: + if multi_attach.not_servable_here(db_dev, snode.get_id()): + # Multi-attach: its state follows the node serving it, not this + # restart (this node built nothing for it). + continue if db_dev.status in [NVMeDevice.STATUS_UNAVAILABLE, NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_CANNOT_ALLOCATE, NVMeDevice.STATUS_READONLY]: db_dev.status = NVMeDevice.STATUS_ONLINE @@ -10335,29 +10610,6 @@ def _check_ftt_allows_node_removal(node_id, db_controller): if node.status not in [StorageNode.STATUS_IN_CREATION, StorageNode.STATUS_REMOVED] ) - # Block suspend/shutdown during rebalancing based on node headroom. - # A cluster needs ndcs+npcs nodes minimum. During rebalancing: - # - With exactly ndcs+npcs nodes: no shutdowns allowed (no headroom) - # - With ndcs+npcs+1 nodes: one shutdown allowed (one spare) - # - With ndcs+npcs+2+ nodes: two shutdowns allowed, etc. - # The number of allowed shutdowns during rebalancing is: - # total_active_nodes - (ndcs + npcs) - # This must be greater than the number of already-not-online nodes. - if cluster.is_re_balancing: - not_online_already = sum( - 1 for node in snodes - if node.get_id() != node_id - and node.status != StorageNode.STATUS_ONLINE - and node.status not in [StorageNode.STATUS_IN_CREATION, StorageNode.STATUS_REMOVED] - ) - headroom = total_active_nodes - (ndcs + npcs) - if headroom <= not_online_already: - return False, ( - f"Cluster is rebalancing with {total_active_nodes} active nodes " - f"({not_online_already} already not online, " - f"need >{ndcs + npcs} for ndcs={ndcs}, npcs={npcs}). " - f"Wait for rebalancing to complete before removing a node." - ) # Count nodes that are not online (excluding the node being removed, # and excluding nodes in creation or already removed). @@ -10533,6 +10785,21 @@ def _check_ftt_allows_node_removal(node_id, db_controller): f"(status: {not_online_node.status})" ) + # Resource floors (simplyblock_core.floors): the cluster AFTER losing + # this node -- its online nodes, their data devices and, when used, its + # failure domains -- must stay at or above every floor. This replaces + # the former "rebalancing headroom" formula (active nodes - (ndcs+npcs)): + # the node floor is n+2k whether or not a rebalance is running. Judged + # last: the FTT / pair / domain refusals above name the concrete node + # the operator has to wait for, which is the more actionable reason. + floor_violations = floors.evaluate(cluster, snodes, exclude_node_id=node_id) + if floor_violations: + return False, ( + f"removing node {node_id} would take the cluster below the resource floors " + f"for the {ndcs}+{npcs} scheme ({total_active_nodes} active nodes): " + + "; ".join(floor_violations) + + ". Bring the other nodes online or add resources first." + ) return True, "" @@ -10589,6 +10856,10 @@ def _detach_remote_controllers_from_peers(snode: StorageNode, db_controller): """ shutting_down_id = snode.get_id() all_peers = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id) + # Multi-attach: controllers to this node's LUNs that another node serves + # lead to that node, not to this one; they stay. + served_elsewhere_ids = {d.get_id() for d in snode.nvme_devices + if multi_attach.served_elsewhere(d, shutting_down_id)} peers = [ p for p in all_peers if p.get_id() != shutting_down_id @@ -10614,6 +10885,8 @@ def _detach_one_peer(peer): for rem_dev in (peer.remote_devices or []): if rem_dev.node_id != shutting_down_id: continue + if rem_dev.get_id() in served_elsewhere_ids: + continue bdev_name = rem_dev.remote_bdev or "" if bdev_name.endswith("n1"): ctrl_names.append(bdev_name[:-2]) @@ -10697,6 +10970,27 @@ def check_node_shutdown_preconditions(node_id, force=False, current_restart_task logger.error(reason) return False, reason + # Resource floors (simplyblock_core.floors): a planned shutdown is refused + # when the cluster without this node would be below a floor (nodes, data + # devices per node and cluster-wide, failure domains). force keeps + # overriding, as it does for every guard below. + if snode.status == StorageNode.STATUS_ONLINE: + try: + _floor_cluster = db_controller.get_cluster_by_id(snode.cluster_id) + except KeyError: + _floor_cluster = None + if _floor_cluster is not None: + _floor_violations = floors.evaluate( + _floor_cluster, db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id), + exclude_node_id=node_id) + if _floor_violations: + reason = (f"shutting down node {node_id} would take the cluster below the " + f"resource floors: " + "; ".join(_floor_violations)) + if force is False: + logger.error(reason) + return False, reason + logger.warning("%s — proceeding with force", reason) + # Guard: no concurrent shutdown + restart (design: mutual exclusion) for peer in db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id): if peer.get_id() != node_id and peer.status == StorageNode.STATUS_RESTARTING: @@ -10803,6 +11097,10 @@ def _broadcast_devices_unavailable(snode): NVMeDevice.STATUS_READONLY, ]: continue + if multi_attach.served_elsewhere(dev, snode.get_id()): + # Multi-attach: another node serves this LUN; it is not lost + # with this node. + continue try: # Default cause (CAUSE_OTHER): a node-driven shutdown # must not count against the per-device flap budget. @@ -19421,6 +19719,12 @@ def create_lvstore(snode: StorageNode, ndcs, npcs, distr_bs, distr_chunk_bs, pag # creation, persisted per stack entry for the same reason. if cluster.page_copy: distrib_params["page_copy"] = True + # raid_devices: every data device is protected below simplyblock, so + # the distrib may put several chunks of a stripe on one device when no + # other device is writable. Persisted like the flags above; the RPC + # sends it only while constants.DISTR_AFFINITY_FLOOR_RPC is on. + if getattr(cluster, "raid_devices", False) is True: + distrib_params["allow_device_colocation"] = True # Failure-domain placement is activated implicitly via the per-node # failure_domain id (>= 0) in the distrib cluster map sent by # distr_send_cluster_map / distr_add_nodes (see get_distr_cluster_map); diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index a403c78c59..6f1fda7df7 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -1656,7 +1656,7 @@ def _unsafe_live_device_reason(dev): return None -def resolve_lblk_entries(configured_entries, host_devices): +def resolve_lblk_entries(configured_entries, host_devices, *, name_fallback=True): """Match the node's configured lblk devices against the live host inventory, SERIAL-FIRST: kernel names shift across reboots, so the stored name is only a fallback for devices without a resolvable serial. Partition @@ -1677,9 +1677,17 @@ def resolve_lblk_entries(configured_entries, host_devices): (mounted, held, or backing the root filesystem) is demoted to `missing` rather than returned as resolved — a fallback match by name/partuuid must clear the same safety bar `filter_eligible_block_devices` enforces - at add-node time, not just "something is attached at that name now".""" + at add-node time, not just "something is attached at that name now". + + ``name_fallback=False`` (multi-attach clusters): an entry with a serial is + matched by serial or partuuid only. An array LUN keeps its identity; when + it is absent the node is fenced from it or it is detached, and its old + kernel name may now be another disk of the host: on the FTT1 bed, after a + hard reset of a node fenced from its LUN, the LUN's name nvme2n1 was the + node's journal volume, which the restart took up as a data device.""" by_path = _index_by_selector(host_devices) - by_serial = {d["serial"]: d for d in host_devices} + # A path of a multipath map shares the map's serial; the map is the device. + by_serial = {d["serial"]: d for d in host_devices if not d.get("multipath_map")} by_name = {d["name"]: d for d in host_devices} by_partuuid = {d["partuuid"].lower(): d for d in host_devices if d.get("partuuid")} @@ -1699,7 +1707,7 @@ def resolve_lblk_entries(configured_entries, host_devices): if live is None and entry.get("partuuid"): live = by_partuuid.get(entry["partuuid"].lower()) matched_by = "partuuid" - if live is None: + if live is None and (name_fallback or not entry.get("serial")): live = by_name.get(entry.get("name")) matched_by = "name" if live is None: @@ -1726,15 +1734,321 @@ def resolve_lblk_entries(configured_entries, host_devices): } if path: resolved_entry["path"] = path + if live.get("wwn"): + # LUN identity of a multi-attach (array LUN) device. + resolved_entry["wwn"] = live["wwn"] if entry.get("type") == "part" or live.get("type") == "part": resolved_entry["type"] = "part" resolved_entry["partuuid"] = live.get("partuuid") or entry.get("partuuid", "") if entry.get("journal"): resolved_entry["journal"] = True + # A LUN the cluster's array plugin created (multi-attach): its + # identity at the array, for multi_attach.mark_lun_devices. + for key in ("lun_wwn", "array_lun_id"): + if entry.get(key): + resolved_entry[key] = entry[key] + # Block-size inputs for a new device's AIO bdev (choose_aio_block_size). + # An agent that predates them sends none; the chooser copes. + for key in ("logical_block_size", "physical_block_size", "start_bytes"): + if live.get(key) is not None: + resolved_entry[key] = live[key] + # The block size this device was formatted with, kept on the config + # entry as a second copy of NVMeDevice.aio_block_size. Only for the + # very same device: a different device in the same slot is new. + if entry.get("aio_block_size") and resolved_entry["serial"] == entry.get("serial"): + resolved_entry["aio_block_size"] = int(entry["aio_block_size"]) + # I/O engine inputs: the kernel name and lsblk type (an agent that + # predates them sends neither; a /dev/mapper/ or /dev/dm- path still + # shows a device-mapper node), the engine the rule gives, and the + # engine recorded on the config entry for the very same device. + if live.get("kname"): + resolved_entry["kname"] = live["kname"] + if live.get("type"): + resolved_entry["dev_type"] = live["type"] + resolved_entry["io_engine"] = lblk_io_engine_for(resolved_entry) + if (entry.get("lblk_io_engine") in constants.LBLK_IO_ENGINES + and resolved_entry["serial"] == entry.get("serial")): + resolved_entry["lblk_io_engine"] = entry["lblk_io_engine"] resolved.append(resolved_entry) return resolved, missing +def choose_aio_block_size(entry) -> int: + """Block size for the AIO bdev of a NEW lblk device (inventory or + resolved entry): AIO_DEFAULT_BLOCK_SIZE (4096) whenever the device allows + it, the device's logical block size otherwise. 4096 is possible when it is + a multiple of the logical block size, the device size is a multiple of it, + and, for a partition whose start is known, the start is 4096-aligned. + + An agent that predates ``logical_block_size`` does not send it. 4096 is + still chosen then when the size allows it: every logical block size up to + 4096 divides it, so SPDK accepts it, and addAioDevices retries a new + device without a block size if SPDK refuses (logical block size above + 4096). When 4096 is not possible, 0 (SPDK picks the logical block size) is + returned instead of a guessed 512, which SPDK would refuse on a 4Kn + device. Never used for a device that already has a record: that one keeps + the block size it was formatted with (recorded_aio_block_size).""" + target = constants.AIO_DEFAULT_BLOCK_SIZE + try: + logical = int(entry.get("logical_block_size") or 0) + except (TypeError, ValueError): + logical = 0 + fallback = max(0, logical) + if logical > target or (logical > 0 and target % logical): + return fallback + size = int(entry.get("size") or 0) + if size <= 0 or size % target: + return fallback + start = entry.get("start_bytes") + if start is not None and int(start) % target: + return fallback + return target + + +def recorded_aio_block_size(snode, serial): + """The block size the AIO bdev of the device with this serial was created + with, from what the control plane recorded about it: its NVMeDevice + record, the node's journal device record, and the node's ``lblk_devices`` + config entry (a plain dict, which a control-plane service that predates + NVMeDevice.aio_block_size keeps intact on a read-modify-write of the + node, while it drops the unknown model field). + + Returns the recorded size, 0 for a device recorded before block sizes were + (legacy: create without a block size, as it was formatted), or None when + nothing is recorded: a new device, which gets choose_aio_block_size.""" + known = False + sizes = [] + devices = getattr(snode, "nvme_devices", None) + for dev in devices if isinstance(devices, list) else []: + if getattr(dev, "serial_number", None) == serial: + known = True + sizes.append(int(getattr(dev, "aio_block_size", 0) or 0)) + jm = getattr(snode, "jm_device", None) + if jm is not None: + data = getattr(jm, "device_data_dict", None) + data = data if isinstance(data, dict) else {} + if data.get("serial_number") == serial or getattr(jm, "serial_number", None) == serial: + known = True + sizes.append(int(data.get("aio_block_size") or 0)) + sizes.append(int(getattr(jm, "aio_block_size", 0) or 0)) + entries = getattr(snode, "lblk_devices", None) + for entry in entries if isinstance(entries, list) else []: + if isinstance(entry, dict) and entry.get("serial") == serial: + sizes.append(int(entry.get("aio_block_size") or 0)) + recorded = next((s for s in sizes if s > 0), 0) + if recorded: + conflicting = sorted({s for s in sizes if s > 0 and s != recorded}) + if conflicting: + logger.warning(f"Device {serial}: conflicting recorded AIO block sizes " + f"{[recorded] + conflicting}; using {recorded}") + return recorded + return 0 if known else None + + +def remember_aio_block_size(snode, serial, block_size): + """Store the block size of a device's AIO bdev on the node's + ``lblk_devices`` entry, next to NVMeDevice.aio_block_size (see + recorded_aio_block_size). 0 (legacy) is never written.""" + entries = getattr(snode, "lblk_devices", None) + if not block_size or not isinstance(entries, list): + return + for entry in entries: + if isinstance(entry, dict) and entry.get("serial") == serial: + entry["aio_block_size"] = int(block_size) + + +def check_aio_block_size(name, bdev, block_size): + """Raise when the AIO bdev does not have the expected (non-zero) block + size. A bdev that reports no block size is not checked.""" + if not block_size or not isinstance(bdev, dict): + return + actual = bdev.get("block_size") + if isinstance(actual, int) and actual != block_size: + raise Exception(f"AIO bdev {name} has block size {actual}, expected " + f"{block_size}; refusing to use it as storage") + + +_DM_PATH_PREFIXES = ("/dev/dm-", "/dev/mapper/") +_DM_BY_ID_PREFIXES = ("dm-name-", "dm-uuid-") + + +def _is_dm_path(path) -> bool: + """A device path that names a device-mapper node: /dev/dm-N, + /dev/mapper/, or a /dev/disk/by-id/dm-name-* / dm-uuid-* link.""" + if not isinstance(path, str) or not path: + return False + if path.startswith(_DM_PATH_PREFIXES): + return True + return (path.startswith("/dev/disk/by-id/") + and path.rsplit("/", 1)[-1].startswith(_DM_BY_ID_PREFIXES)) + + +def lblk_io_engine_for(dev) -> str: + """The I/O engine the rule gives a block device (inventory dict, + resolved entry, or a dict of paths): "uring" for a device-mapper node, + "aio" for everything else (see constants.LBLK_IO_ENGINE_URING). + + A device-mapper node is one whose kernel name starts with "dm-" (this + includes a partition kpartx mapped from a multipath map), whose lsblk + TYPE is one of LBLK_DM_TYPES, or, for an agent that predates ``kname`` + and the type, whose path is /dev/dm-*, /dev/mapper/* or a by-id + dm-name-* / dm-uuid-* link. Whole disks, sd*/nvme* partitions and md + arrays stay on aio: md honours RWF_NOWAIT (EAGAIN is retried by SPDK).""" + if not isinstance(dev, dict): + return constants.LBLK_IO_ENGINE_AIO + for key in ("kname", "name"): + if str(dev.get(key) or "").startswith("dm-"): + return constants.LBLK_IO_ENGINE_URING + for key in ("dev_type", "type"): + if dev.get(key) in constants.LBLK_DM_TYPES: + return constants.LBLK_IO_ENGINE_URING + for key in ("path", "filename", "device_path", "current_path", "by_id", "by_id_path"): + if _is_dm_path(dev.get(key)): + return constants.LBLK_IO_ENGINE_URING + return constants.LBLK_IO_ENGINE_AIO + + +def recorded_lblk_io_engine(snode, serial): + """The I/O engine recorded for the device with this serial: its + NVMeDevice record, the node's journal device record and the node's + ``lblk_devices`` entry (see recorded_aio_block_size for why all three). + "uring" when any of them says so (a device is never moved back from + uring implicitly), "aio" when it is recorded otherwise, None when nothing + is recorded.""" + engines = [] + devices = getattr(snode, "nvme_devices", None) + for dev in devices if isinstance(devices, list) else []: + if getattr(dev, "serial_number", None) == serial: + engines.append(getattr(dev, "lblk_io_engine", None)) + jm = getattr(snode, "jm_device", None) + if jm is not None: + data = getattr(jm, "device_data_dict", None) + data = data if isinstance(data, dict) else {} + if data.get("serial_number") == serial or getattr(jm, "serial_number", None) == serial: + engines.append(data.get("lblk_io_engine")) + engines.append(getattr(jm, "lblk_io_engine", None)) + entries = getattr(snode, "lblk_devices", None) + for entry in entries if isinstance(entries, list) else []: + if isinstance(entry, dict) and entry.get("serial") == serial: + engines.append(entry.get("lblk_io_engine")) + engines = [e for e in engines if e in constants.LBLK_IO_ENGINES] + if constants.LBLK_IO_ENGINE_URING in engines: + return constants.LBLK_IO_ENGINE_URING + return constants.LBLK_IO_ENGINE_AIO if engines else None + + +def choose_lblk_io_engine(recorded, derived, what="") -> str: + """The engine to create a device's base bdev with, from the recorded + engine (None: nothing recorded) and the one the rule gives now + (lblk_io_engine_for). The engine does not change the on-disk format, so + it may be re-derived; the recorded one is kept, except that the rule's + uring wins over a recorded aio: bdev_aio cannot serve a device-mapper + node at all (e.g. a device recorded before the engine existed).""" + if derived == constants.LBLK_IO_ENGINE_URING: + if recorded == constants.LBLK_IO_ENGINE_AIO: + logger.warning(f"{what}: recorded I/O engine is aio, but the device is a " + f"device-mapper node, which bdev_aio cannot serve; using uring") + return constants.LBLK_IO_ENGINE_URING + if recorded == constants.LBLK_IO_ENGINE_URING: + return constants.LBLK_IO_ENGINE_URING + return constants.LBLK_IO_ENGINE_AIO + + +def lblk_io_engine_of_device(snode, dev, live=None, filename=None) -> str: + """The engine for (re-)creating the base bdev of an existing lblk device + record ``dev``: its recorded engine (the record itself, the journal + record, the node's config entry), with the rule applied to the live + inventory dict ``live`` (if any), the ``filename`` the bdev is opened on + and the stored paths.""" + serial = getattr(dev, "serial_number", None) + engines = [recorded_lblk_io_engine(snode, serial), getattr(dev, "lblk_io_engine", None)] + engines = [e for e in engines if e in constants.LBLK_IO_ENGINES] + recorded = None + if engines: + recorded = (constants.LBLK_IO_ENGINE_URING + if constants.LBLK_IO_ENGINE_URING in engines + else constants.LBLK_IO_ENGINE_AIO) + probe = {"device_path": getattr(dev, "device_path", "") or "", + "by_id_path": getattr(dev, "by_id_path", "") or ""} + if isinstance(live, dict): + probe.update(live) + probe["filename"] = filename or "" + return choose_lblk_io_engine(recorded, lblk_io_engine_for(probe), + what=f"Device {serial}") + + +def remember_lblk_io_engine(snode, serial, io_engine): + """Store a device's I/O engine on the node's ``lblk_devices`` entry, next + to NVMeDevice.lblk_io_engine (see recorded_lblk_io_engine).""" + entries = getattr(snode, "lblk_devices", None) + if io_engine not in constants.LBLK_IO_ENGINES or not isinstance(entries, list): + return + for entry in entries: + if isinstance(entry, dict) and entry.get("serial") == serial: + entry["lblk_io_engine"] = io_engine + + +def bdev_io_engine(bdev): + """The engine of a live bdev (a bdev_get result) by its driver-specific + info: "aio", "uring", or None when it does not say.""" + driver = bdev.get("driver_specific") if isinstance(bdev, dict) else None + if isinstance(driver, dict): + for engine in (constants.LBLK_IO_ENGINE_URING, constants.LBLK_IO_ENGINE_AIO): + if engine in driver: + return engine + return None + + +def _lblk_bdev_rpc(rpc_client, io_engine, op): + """The bdev_aio_ or bdev_uring_ method of ``rpc_client``.""" + if io_engine == constants.LBLK_IO_ENGINE_URING: + return getattr(rpc_client, f"bdev_uring_{op}"), f"bdev_uring_{op}" + return getattr(rpc_client, f"bdev_aio_{op}"), f"bdev_aio_{op}" + + +def delete_lblk_bdev(rpc_client, name, io_engine=constants.LBLK_IO_ENGINE_AIO): + """bdev_aio_delete or bdev_uring_delete, by the device's I/O engine.""" + delete, _ = _lblk_bdev_rpc(rpc_client, io_engine, "delete") + return delete(name) + + +def rescan_lblk_bdev(rpc_client, name, io_engine=constants.LBLK_IO_ENGINE_AIO): + """bdev_aio_rescan or bdev_uring_rescan (re-read the size of the block + device), by the device's I/O engine.""" + rescan, _ = _lblk_bdev_rpc(rpc_client, io_engine, "rescan") + return rescan(name) + + +def create_aio_bdev(rpc_client, name, filename, block_size=0, new_device=False, + verify=True, io_engine=constants.LBLK_IO_ENGINE_AIO): + """Create the base bdev of an lblk device: bdev_aio_create, or for + ``io_engine`` "uring" bdev_uring_create (same arguments, same block-size + rules), with ``block_size`` (0: none, SPDK uses the logical block size), + then (``verify``) check that the bdev has that block size; a caller that + reads the bdev anyway passes verify=False and runs check_aio_block_size + itself. Returns the block size requested. Raises when the bdev cannot be + created or its block size differs from the one requested: building + alceml over a bdev of another block size would make the device's data + unreadable. Only for a ``new_device`` (nothing formatted on it yet) a + refused block size is retried without one.""" + create, rpc_name = _lblk_bdev_rpc(rpc_client, io_engine, "create") + if block_size: + ret = create(name, filename, block_size=block_size) + if not ret and new_device: + logger.warning(f"{rpc_name} {name} ({filename}) with block size " + f"{block_size} failed on {rpc_client.host}; retrying with " + f"the device's logical block size") + block_size = 0 + ret = create(name, filename) + else: + ret = create(name, filename) + if not ret: + raise Exception(f"{rpc_name} failed for {name} ({filename}) on {rpc_client.host}") + if verify: + check_aio_block_size(name, rpc_client.bdev_get(name), block_size) + return block_size + + def addAioDevices(rpc_client, snode, blk_entries): """lblk-mode sibling of addNvmeDevices: create one SPDK AIO bdev per resolved block device and model it as an NVMeDevice with @@ -1745,27 +2059,57 @@ def addAioDevices(rpc_client, snode, blk_entries): next_physical_label = snode.physical_label for entry in blk_entries: bdev_name = aio_bdev_name_for_serial(entry["serial"]) + # A device the cluster knows is re-created with the block size it was + # formatted with (0: legacy, none); only a new one gets the default. + recorded = recorded_aio_block_size(snode, entry["serial"]) + if recorded is None and entry.get("aio_block_size"): + recorded = int(entry["aio_block_size"]) + # The I/O engine: the recorded one, unless the rule says uring (a + # device-mapper node, which bdev_aio cannot serve). + recorded_engine = recorded_lblk_io_engine(snode, entry["serial"]) + if recorded_engine is None and entry.get("lblk_io_engine") in constants.LBLK_IO_ENGINES: + recorded_engine = entry["lblk_io_engine"] + io_engine = choose_lblk_io_engine( + recorded_engine, entry.get("io_engine") or lblk_io_engine_for(entry), + what=f"Block device {entry.get('name')} ({bdev_name})") ret = rpc_client.bdev_get(bdev_name) if not ret: # A path the caller selected the device by is opened as given. # Otherwise prefer the by-id path as the filename so a udev rename # between resolution and create cannot swap devices under us. filename = entry.get("path") or entry.get("by_id") or entry["current_path"] - ret = rpc_client.bdev_aio_create(bdev_name, filename) - if not ret: - raise Exception( - f"bdev_aio_create failed for {bdev_name} ({filename}) " - f"on {rpc_client.host}") + if recorded is None: + expected = create_aio_bdev(rpc_client, bdev_name, filename, + choose_aio_block_size(entry), + new_device=True, verify=False, + io_engine=io_engine) + else: + expected = create_aio_bdev(rpc_client, bdev_name, filename, + recorded, verify=False, + io_engine=io_engine) + else: + expected = recorded + # A bdev that is still there is used as it is; record its engine. + live_engine = bdev_io_engine(ret) + if live_engine and live_engine != io_engine: + logger.warning(f"Base bdev {bdev_name} exists as a {live_engine} bdev, " + f"expected {io_engine}; using it as it is") + io_engine = live_engine rpc_client.bdev_examine(bdev_name) rpc_client.bdev_wait_for_examine() bdev = rpc_client.bdev_get(bdev_name) if not bdev: raise Exception(f"AIO bdev {bdev_name} not found after create on {rpc_client.host}") + check_aio_block_size(bdev_name, bdev, expected) total_size = bdev['block_size'] * bdev['num_blocks'] if total_size == 0: logger.warning(f"Skipping zero-size block device {entry['name']} ({bdev_name})") continue + # A new device is formatted at the block size its bdev has now. + aio_block_size = int(bdev['block_size']) if recorded is None else recorded + remember_aio_block_size(snode, entry["serial"], aio_block_size) + remember_lblk_io_engine(snode, entry["serial"], io_engine) # Queue-depth sampling feeds the control-plane hung-IO watchdog # (AIO has no bdev_nvme-style timeout/action_on_timeout). @@ -1787,6 +2131,8 @@ def addAioDevices(rpc_client, snode, blk_entries): 'nvme_bdev': bdev_name, 'nvme_controller': "", 'bdev_type': "aio", + 'aio_block_size': aio_block_size, + 'lblk_io_engine': io_engine, 'device_path': entry["current_path"], 'by_id_path': entry.get("path") or entry.get("by_id", ""), 'node_id': snode.get_id(), @@ -2436,6 +2782,21 @@ def previous_lblk_dedicated_journal_serials(config_paths=None) -> set: return serials +def lblk_journal_only_layout(lblk_entries, socket=0): + """``sn configure --lblk-journal-only``: the node's one selected disk or + partition becomes its journal, flagged, and is used whole (a disk is + never partitioned). The node then has no local data device: on a + multi-attach cluster whose array plugin creates each node's data LUN at + add-node, that LUN is its data device. Raises ValueError unless exactly + one disk or partition is selected.""" + if len(lblk_entries) != 1: + raise ValueError( + f"--lblk-journal-only needs exactly one disk or partition for the node on socket " + f"{socket}; selected: {sorted(lblk_entries)}") + name, entry = next(iter(lblk_entries.items())) + return {name: dict(entry, journal=True)} + + def lblk_node_journal_layout(lblk_entries, jm_percent=3, journal_device=False, keep_serials=()): """Give one storage node's devices their journal. @@ -2491,7 +2852,11 @@ def lblk_node_journal_layout(lblk_entries, jm_percent=3, journal_device=False, def node_config_device_count(node) -> int: """Number of storage devices a node-config entry carries — ssd_pcis for - nvme mode, lblk_devices for lblk mode.""" + nvme mode, lblk_devices for lblk mode. A node configured with + --lblk-array-only carries none: its array plugin creates its journal LUN + and data LUN at add-node, so it counts those two.""" + if node.get("lblk_array_only"): + return 2 return len(node.get("lblk_devices") or []) or len(node.get("ssd_pcis") or []) @@ -2508,15 +2873,16 @@ def lblk_device_serials(lblk_devices) -> set[str]: return {e["serial"] for e in (lblk_devices or []) if e.get("serial")} -def node_config_min_sys_memory(node) -> int: +def node_config_min_sys_memory(node, extra_capacity=0) -> int: """Minimum system memory for a node-config entry: sys_memory_for_capacity of its total device capacity. lblk entries carry their sizes; nvme entries carry ssd_capacity, measured by generate_configs before it unbound the devices, since nvme-cli no longer sees them afterwards. An entry written - before that field existed goes through nvme-cli.""" + before that field existed goes through nvme-cli. ``extra_capacity`` + (lblk only): the LUNs an array plugin creates for the node at add-node.""" lblk = node.get("lblk_devices") or [] - if lblk: - capacity = sum(int(e.get("size") or 0) for e in lblk) + if lblk or node.get("lblk_array_only"): + capacity = sum(int(e.get("size") or 0) for e in lblk) + int(extra_capacity or 0) elif node.get("ssd_capacity"): capacity = int(node["ssd_capacity"]) else: @@ -2876,6 +3242,8 @@ def regenerate_config(new_config, old_config, force=False): old_config["nodes"][i]["ssd_pcis"] = new_config["nodes"][i]["ssd_pcis"] if new_config["nodes"][i].get("lblk_devices") is not None: old_config["nodes"][i]["lblk_devices"] = new_config["nodes"][i]["lblk_devices"] + if "lblk_array_only" in new_config["nodes"][i]: + old_config["nodes"][i]["lblk_array_only"] = new_config["nodes"][i]["lblk_array_only"] old_config["nodes"][i]["nic_ports"] = new_config["nodes"][i]["nic_ports"] for nic in old_config["nodes"][i]["nic_ports"]: if nic not in all_nics: @@ -2937,7 +3305,14 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a nics = detect_nics() lblk_mode = lblk_selection is not None lblk_entries: dict = {} - if lblk_mode: + array_only = bool(lblk_mode and lblk_selection.get("array_only")) + if array_only: + # sn configure --lblk-array-only: no local device at all. The array + # plugin creates the node's journal LUN and data LUN at add-node. + lblk_units: dict = {} + lblk_keep_serials: set = set() + nvmes: dict = {} + elif lblk_mode: # lblk cluster mode: eligible Linux block devices instead of NVMe # PCIe controllers. No driver unbind, no formatting here (--force # only marks partitioned disks eligible; the wipe happens at @@ -3108,7 +3483,10 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a node_info["number_of_distribs"] = number_of_distribs nvme_neg1_list = all_nvmes_neg1_per_node[node_index] - if lblk_mode: + if array_only: + node_info["lblk_devices"] = [] + node_info["lblk_array_only"] = True + elif lblk_mode: node_entries = {} for unit in list(nvme_neg1_list) + list(nvme_per_core_group[idx]): for dev_name in lblk_units[unit]: @@ -3116,11 +3494,15 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a entry.pop("_pair", None) node_entries[dev_name] = entry try: - node_entries = lblk_node_journal_layout( - node_entries, jm_percent=jm_percent, - journal_device=bool(lblk_selection.get("journal_device")), - keep_serials=lblk_keep_serials) - if node_entries and len(node_entries) < constants.LBLK_MIN_DEVICES_PER_NODE: + if lblk_selection.get("journal_only"): + node_entries = lblk_journal_only_layout(node_entries, nid) + else: + node_entries = lblk_node_journal_layout( + node_entries, jm_percent=jm_percent, + journal_device=bool(lblk_selection.get("journal_device")), + keep_serials=lblk_keep_serials) + if (node_entries and not lblk_selection.get("journal_only") + and len(node_entries) < constants.LBLK_MIN_DEVICES_PER_NODE): raise ValueError( f"lblk mode requires at least {constants.LBLK_MIN_DEVICES_PER_NODE} " f"partitions or SSDs per node; only {len(node_entries)} unit(s) " @@ -3544,7 +3926,14 @@ def validate_node_config(node): # Exactly one device source: PCIe SSDs (nvme mode) or Linux block # devices (lblk mode). Both empty, or both populated, is a broken config. lblk_devices = node.get("lblk_devices") or [] - if bool(node["ssd_pcis"]) == bool(lblk_devices): + if node.get("lblk_array_only"): + # sn configure --lblk-array-only: no local device; the array plugin + # creates the node's journal and data LUNs at add-node. + if node["ssd_pcis"] or lblk_devices: + logger.error(f"A node configured with --lblk-array-only carries no devices, " + f"node {node.get('socket')} does") + return False + elif bool(node["ssd_pcis"]) == bool(lblk_devices): logger.error( f"Node config must carry exactly one non-empty device list of " f"'ssd_pcis' / 'lblk_devices' in node: {node.get('socket')}") @@ -3566,7 +3955,11 @@ def validate_node_config(node): f"'size' in node: {node.get('socket')}") return False if lblk_devices: - if len(lblk_devices) < constants.LBLK_MIN_DEVICES_PER_NODE: + # sn configure --lblk-journal-only: the node's one entry is its flagged + # journal disk; its data device is the array LUN added at add-node, + # where the minimum is checked again with that LUN counted. + journal_only = len(lblk_devices) == 1 and bool(lblk_devices[0].get("journal")) + if len(lblk_devices) < constants.LBLK_MIN_DEVICES_PER_NODE and not journal_only: logger.error( f"lblk mode requires at least {constants.LBLK_MIN_DEVICES_PER_NODE} " f"partitions or SSDs per node; node {node.get('socket')} carries " diff --git a/simplyblock_core/utils/nvme.py b/simplyblock_core/utils/nvme.py index 1ed6cce416..8b533e5e32 100644 --- a/simplyblock_core/utils/nvme.py +++ b/simplyblock_core/utils/nvme.py @@ -130,6 +130,12 @@ def build_nvme_connect_entry( # reconnect 2 s (docs/design/two-node-arbitration.md, client # requirements). Clusters without arbitration keep the defaults. keep_alive_tmo = constants.TWO_NODE_CLIENT_KEEP_ALIVE_TO + elif getattr(cluster, "multi_attach", False) is True: + # Multi-attach (1+0): a node lost without closing its connections + # leaves its own volumes' paths dead until the host's keep-alive + # notices (constants.MULTI_ATTACH_CLIENT_KEEP_ALIVE_TO). Regular + # clusters keep the defaults. + keep_alive_tmo = constants.MULTI_ATTACH_CLIENT_KEEP_ALIVE_TO client_data_nic_str = f"--host-iface={cluster.client_data_nic}" if cluster.client_data_nic else "" tls_str = host_auth_str = "" if host_entry: diff --git a/simplyblock_web/api/internal/_node_info.py b/simplyblock_web/api/internal/_node_info.py index e82f4ddca2..be3ccf06f0 100644 --- a/simplyblock_web/api/internal/_node_info.py +++ b/simplyblock_web/api/internal/_node_info.py @@ -158,16 +158,23 @@ def _google_info() -> dict | None: def _amazon_info() -> dict | None: + # IMDSv2 directly: ec2_metadata.EC2Metadata no longer takes a session + # (TypeError, so this returned None on every AWS instance). try: - import ec2_metadata - session = requests.session() - data = ec2_metadata.EC2Metadata(session=session).instance_identity_document # type: ignore[call-arg] + token = requests.put("http://169.254.169.254/latest/api/token", timeout=2, + headers={"X-aws-ec2-metadata-token-ttl-seconds": "60"}).text + data = requests.get("http://169.254.169.254/latest/dynamic/instance-identity/document", + timeout=2, headers={"X-aws-ec2-metadata-token": token}).json() return { "id": data["instanceId"], "type": data["instanceType"], "cloud": "amazon", "ip": data["privateIp"], "public_ip": "", + # Multi-attach clusters on EBS (simplyblock_core.array.aws_ebs): + # volumes attach only within their zone. + "availability_zone": data.get("availabilityZone", ""), + "region": data.get("region", ""), } except Exception: return None diff --git a/simplyblock_web/api/internal/storage_node/docker.py b/simplyblock_web/api/internal/storage_node/docker.py index b90894bd3e..80a70b317d 100644 --- a/simplyblock_web/api/internal/storage_node/docker.py +++ b/simplyblock_web/api/internal/storage_node/docker.py @@ -92,6 +92,45 @@ class SPDKParams(BaseModel): mcp_max_unavailable: int | None = Field(None) # OpenShift-only (MCP); ignored here +#: Where the agent keeps the last log lines of an SPDK container that died on +#: its own, before a restart removes it (host-mounted /etc/simplyblock). +SPDK_CRASH_LOG_DIR = os.environ.get("SPDK_CRASH_LOG_DIR", "/etc/simplyblock/spdk-crash-logs") +SPDK_CRASH_LOG_LINES = 30000 +SPDK_CRASH_LOG_KEEP = 10 + + +def _preserve_exited_spdk_logs(client, name): + """Save the log tail and exit state of ``name`` if its container already + exited -- SPDK died on its own -- before a restart removes it with its + logs (2026-11-20 FTT2 bed: three SPDKs died in a cascade, every trace of + why went with the containers). A running container (a deliberate kill) + is left alone; best-effort, never raises.""" + try: + container = client.containers.get(name) + state = (container.attrs or {}).get("State") or {} + if state.get("Status") not in ("exited", "dead"): + return None + os.makedirs(SPDK_CRASH_LOG_DIR, exist_ok=True) + stamp = time.strftime("%Y%m%d-%H%M%S", time.gmtime()) + path = os.path.join(SPDK_CRASH_LOG_DIR, f"{name.strip('/')}-{stamp}.log") + logs = container.logs(tail=SPDK_CRASH_LOG_LINES, timestamps=True) + with open(path, "wb") as f: + exit_state = {k: state.get(k) for k in ( + "ExitCode", "OOMKilled", "Error", "StartedAt", "FinishedAt")} + f.write(json.dumps(exit_state).encode() + os.linesep.encode()) + f.write(logs if isinstance(logs, bytes) else str(logs).encode()) + kept = sorted(Path(SPDK_CRASH_LOG_DIR).glob("spdk_*.log"), key=lambda q: q.stat().st_mtime) + for old in kept[:-SPDK_CRASH_LOG_KEEP]: + old.unlink(missing_ok=True) + logger.warning("SPDK container %s had exited (code %s, OOM %s); its log kept in %s", + name, state.get("ExitCode"), state.get("OOMKilled"), path) + return path + except Exception as exc: # noqa: BLE001 + logger.info("no crash log kept for %s: %s", name, exc) + return None + + + @api.post('/spdk_process_start', responses={ 200: {'content': {'application/json': {'schema': utils.response_schema({ 'type': 'boolean' @@ -107,6 +146,7 @@ def spdk_process_start(body: SPDKParams): node_docker = get_docker_client(timeout=60 * 3) spdk_names = [f"/spdk_{body.rpc_port}", f"/spdk_proxy_{body.rpc_port}"] for name in spdk_names: + _preserve_exited_spdk_logs(node_docker, name) core_utils.remove_container(node_docker, name, graceful_timeout=0) # Confirm the previous instances are actually gone before we start a fresh @@ -263,6 +303,7 @@ def spdk_process_kill(query: utils.RPCPortParams): # SIGKILLing whatever currently answers to the name. targets = [] for name in names: + _preserve_exited_spdk_logs(client, name) try: targets.append(client.containers.get(name)) except NotFound: @@ -332,6 +373,7 @@ def spdk_process_cleanup(query: utils.RPCPortParams): names = [f"/spdk_{query.rpc_port}", f"/spdk_proxy_{query.rpc_port}"] ok = True for name in names: + _preserve_exited_spdk_logs(client, name) try: container = client.containers.get(name) except NotFound: @@ -657,6 +699,22 @@ def wipe_block_device(body: _WipeBlockDeviceParams): return utils.get_response(True) +class _LunInfoParams(BaseModel): + wwn: str + + +@api.post('/lun_info', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'array', + 'items': {'type': 'object', 'additionalProperties': True}, + })}}}, +}) +def lun_info(body: _LunInfoParams): + """Multi-attach array LUN mode: the local disks carrying LUN ``wwn``, with + whether block 0 holds the alceml signature. Read-only.""" + return utils.get_response(node_utils.find_lun_devices(body.wwn)) + + class _RescanBlockDeviceParams(BaseModel): device_path: str @@ -681,6 +739,22 @@ def rescan_block_device(body: _RescanBlockDeviceParams): return utils.get_response(None, str(e)) +@api.post('/scan_new_luns', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'object', + 'properties': {'scanned': {'type': 'array', 'items': {'type': 'string'}}}, + })}}}, +}) +def scan_new_luns(): + """Look for LUNs / namespaces a storage array newly presented to this + host (SCSI host scan, NVMe namespace rescan). Existing devices are not + touched.""" + try: + return utils.get_response(node_utils.scan_for_new_luns()) + except OSError as e: + return utils.get_response(None, str(e)) + + class _DeviceSmartQuery(BaseModel): device_path: str @@ -772,6 +846,9 @@ def get_info(): "network_interface": core_utils.get_nics_data(), "cloud_instance": node_info["cloud_info"], + # SAN initiators (IQN, FC WWPNs, NVMe host NQN, SDC GUID): how a storage + # array knows this host (simplyblock_core.array.san). + "san_initiators": node_utils.get_san_initiators(), "lsblk": get_node_lsblk(), "nodes_config": get_nodes_config(), diff --git a/simplyblock_web/api/internal/storage_node/kubernetes.py b/simplyblock_web/api/internal/storage_node/kubernetes.py index e8d58b5536..64b1c8ae41 100644 --- a/simplyblock_web/api/internal/storage_node/kubernetes.py +++ b/simplyblock_web/api/internal/storage_node/kubernetes.py @@ -316,6 +316,9 @@ def get_info(): "host_ip": os.environ.get("HOST_IP", ""), "cloud_instance": node_info["cloud_info"], + # SAN initiators (IQN, FC WWPNs, NVMe host NQN, SDC GUID): how a storage + # array knows this host (simplyblock_core.array.san). + "san_initiators": node_utils.get_san_initiators(), "nodes_config": get_nodes_config(), }) @@ -350,6 +353,22 @@ def wipe_block_device(body: _WipeBlockDeviceParams): return utils.get_response(True) +class _LunInfoParams(BaseModel): + wwn: str + + +@api.post('/lun_info', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'array', + 'items': {'type': 'object', 'additionalProperties': True}, + })}}}, +}) +def lun_info(body: _LunInfoParams): + """Multi-attach array LUN mode: the local disks carrying LUN ``wwn``, with + whether block 0 holds the alceml signature. Read-only.""" + return utils.get_response(node_utils.find_lun_devices(body.wwn)) + + class _RescanBlockDeviceParams(BaseModel): device_path: str @@ -374,6 +393,22 @@ def rescan_block_device(body: _RescanBlockDeviceParams): return utils.get_response(None, str(e)) +@api.post('/scan_new_luns', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'object', + 'properties': {'scanned': {'type': 'array', 'items': {'type': 'string'}}}, + })}}}, +}) +def scan_new_luns(): + """Look for LUNs / namespaces a storage array newly presented to this + host (SCSI host scan, NVMe namespace rescan). Existing devices are not + touched.""" + try: + return utils.get_response(node_utils.scan_for_new_luns()) + except OSError as e: + return utils.get_response(None, str(e)) + + class _DeviceSmartQuery(BaseModel): device_path: str diff --git a/simplyblock_web/api/v1/cluster.py b/simplyblock_web/api/v1/cluster.py index 46119e8ecf..5a33e9f9ae 100644 --- a/simplyblock_web/api/v1/cluster.py +++ b/simplyblock_web/api/v1/cluster.py @@ -51,6 +51,9 @@ def add_cluster(): max_queue_size = cl_data.get('max_queue_size', 128) inflight_io_threshold = cl_data.get('inflight_io_threshold', 4) strict_node_anti_affinity = cl_data.get('strict_node_anti_affinity', False) + raid_devices = bool(cl_data.get('raid_devices', False)) + multi_attach = bool(cl_data.get('multi_attach', False)) + ftt = cl_data.get('ftt') is_single_node = cl_data.get('is_single_node', False) client_data_nic = cl_data.get('client_data_nic', "") max_fault_tolerance = min(distr_npcs, 2) if distr_npcs >= 1 else 1 @@ -67,6 +70,10 @@ def add_cluster(): cr_name, cr_namespace, cr_plural, fabric, client_data_nic=client_data_nic, max_fault_tolerance=max_fault_tolerance, backup_config=backup_config, nvmf_base_port=nvmf_base_port, rpc_base_port=rpc_base_port, snode_api_port=snode_api_port, + raid_devices=raid_devices, + multi_attach=multi_attach, + ftt=ftt, + multi_attach_warm_standby=bool(cl_data.get('multi_attach_warm_standby', False)), kms_settings=cl_data.get('kms_settings'), lvol_geometry_enabled=lvol_geometry_enabled, # Absent inherits the prefix of a cluster already on this control plane. diff --git a/simplyblock_web/api/v2/_dtos.py b/simplyblock_web/api/v2/_dtos.py index c946c655bf..cfce8359e1 100644 --- a/simplyblock_web/api/v2/_dtos.py +++ b/simplyblock_web/api/v2/_dtos.py @@ -212,6 +212,9 @@ class ClusterDTO(BaseModel): atomic_4k: bool page_copy: bool = False md_journal: bool = False + raid_devices: bool = False + multi_attach: bool = False + multi_attach_warm_standby: bool = False secret: SecretStr tls_enabled: bool max_fault_tolerance: int @@ -264,6 +267,9 @@ def from_model(model: Cluster, stat_obj: StatsObject | None = None): atomic_4k=model.atomic_4k, page_copy=model.page_copy, md_journal=model.md_journal, + raid_devices=model.raid_devices, + multi_attach=model.multi_attach, + multi_attach_warm_standby=getattr(model, "multi_attach_warm_standby", False) is True, secret=model.secret, tls_enabled=model.tls, max_fault_tolerance=model.max_fault_tolerance, diff --git a/simplyblock_web/api/v2/cluster/__init__.py b/simplyblock_web/api/v2/cluster/__init__.py index e608563ccb..b8b06893d7 100644 --- a/simplyblock_web/api/v2/cluster/__init__.py +++ b/simplyblock_web/api/v2/cluster/__init__.py @@ -32,6 +32,7 @@ from .._sse import WATCH_RESPONSES, WatchParam, sse_response from .alert import api as alert_api from .arbitration import api as arbitration_api +from .array import api as array_api from .backup import api as backup_api from .consistency_group import api as consistency_group_api from .group_backup import api as group_backup_api @@ -77,6 +78,17 @@ class ClusterParams(BaseModel): inflight_io_threshold: int = 4 enable_node_affinity: bool = False strict_node_anti_affinity: bool = False + # Every data device is protected below simplyblock (hardware RAID LUN, md + # or dm RAID): data-device floor n+k instead of n+2k. Immutable. + raid_devices: bool = False + # Multi-attach array LUN mode (1+0, FTT from LUN takeover). Requires + # raid_devices, ha_type 'ha' and ftt. Immutable. + multi_attach: bool = False + #: Node failures to tolerate in multi-attach mode; only with multi_attach. + ftt: Literal[1, 2] | None = None + #: Multi-attach only: keep warm standby alcemls of every LUN on its + #: attached non-serving nodes (fast takeover; costs page-map hugepages). + multi_attach_warm_standby: bool = False is_single_node: bool = False fabric: str = "tcp" cr_name: str = "" @@ -403,33 +415,63 @@ def rebalance_cluster( cluster: Cluster) -> Response: return Response(status_code=204) -class DeviceRebalanceDTO(BaseModel): - weights_updated: bool - started: bool - reason: str +class DeviceGrowthEntryDTO(BaseModel): + device_id: str + old_size: int + new_size: int + backend_size: int + unused_size: int + placement_size: int class DeviceGrowthDTO(BaseModel): - devices: builtins.list[dict] + grown: bool + factor: float + devices: builtins.list[DeviceGrowthEntryDTO] errors: dict[str, str] - uniform: bool - rebalance: DeviceRebalanceDTO | None = None + reason: str + shrunk: builtins.list[str] = [] + luns: dict[str, dict] | None = None @instance_api.post('/grow-devices', name='clusters:grow_devices', response_model=DeviceGrowthDTO) -def grow_devices(cluster: Cluster, rebalance: bool = True) -> DeviceGrowthDTO: - """Grow every online lblk data device of the cluster to the size of its - block device, one device at a time. Used after the LUNs of the cluster - were grown on the storage array. If they have grown unevenly, the device - weights are set to the new sizes and the cluster is rebalanced, unless - ``rebalance`` is false.""" +def grow_devices(cluster: Cluster, + lun_factor: Annotated[float | None, Query(gt=1)] = None, + lun_size: Annotated[int | None, Query(gt=0)] = None) -> DeviceGrowthDTO: + """Grow all lblk data devices of the cluster online by the same factor. + + Every block device is rescanned (for example, after its LUN was grown on + the storage array), and every device grows by the smallest growth factor + of all of them; the rest of a larger block device stays unused + (`unused_size`). If one block device did not grow, nothing grows + (`grown` false, `reason`). Device weights and data placement never change. + A block device smaller than its device is refused, listed in `shrunk` + and `errors`, and raises the device_shrunk alert. + + On a cluster whose array plugin creates its LUNs, every LUN is grown at + the array first: by `lun_factor`, or to `lun_size` bytes (a LUN already + that large is left alone, so the call can be repeated after a failure; + nodes added later get LUNs of that size). + """ if cluster.device_mode != constants.DEVICE_MODE_LBLK: raise HTTPException(409, 'Online device growth needs a cluster in lblk device mode') - return DeviceGrowthDTO(**device_controller.grow_cluster_devices(cluster.get_id(), rebalance=rebalance)) + if lun_factor is not None and lun_size is not None: + raise HTTPException(400, 'Give lun_factor or lun_size, not both') + extra: dict = {} + if lun_factor is not None: + extra["lun_factor"] = lun_factor + if lun_size is not None: + extra["lun_size"] = lun_size + try: + result = device_controller.grow_cluster_devices(cluster.get_id(), **extra) + except ValueError as e: + raise HTTPException(409, str(e)) + return DeviceGrowthDTO(**result) instance_api.include_router(alert_api, prefix='/alerts') instance_api.include_router(arbitration_api, prefix='/arbitration') +instance_api.include_router(array_api, prefix='/arrays') instance_api.include_router(storage_node_api, prefix='/storage-nodes') instance_api.include_router(task_api, prefix='/tasks') instance_api.include_router(pool_api, prefix='/storage-pools') diff --git a/simplyblock_web/api/v2/cluster/array.py b/simplyblock_web/api/v2/cluster/array.py new file mode 100644 index 0000000000..be53e99421 --- /dev/null +++ b/simplyblock_web/api/v2/cluster/array.py @@ -0,0 +1,104 @@ +"""The storage array of a multi-attach cluster (array LUN mode, phase 4). + +simplyblock creates, presents, grows, fences and deletes the cluster's LUNs +through the array plugin registered here. Credentials are wrapped by +the cluster's KMS (or stay in an external Vault KV / KMIP secret the request +references) and are never returned; everything else is kept on the cluster +record. See simplyblock_core.controllers.array_controller. +""" + +from typing import Annotated, Any, Literal + +from fastapi import APIRouter, HTTPException, Response +from pydantic import BaseModel, Field, SecretStr + +from simplyblock_core.controllers import array_controller + +from .. import util +from .._dependencies import Cluster + +api = APIRouter() + + +class ArrayParams(BaseModel): + plugin: Annotated[str, Field(min_length=1, description="Plugin name (entry point in simplyblock.array_plugins), e.g. aws-ebs")] + endpoint: str = "" + options: dict[str, Any] = Field(default_factory=dict) + # Wrapped by the cluster's KMS, never stored in the clear; masked in every + # representation of this model. + credentials: dict[str, SecretStr] = Field(default_factory=dict) + credential_ref: Annotated[dict[str, Any], Field(description=( + "Instead of credentials: a secret kept outside simplyblock, " + '{"kind": "vault-kv", "mount": ..., "path": ..., "version": ...} or ' + '{"kind": "kmip-secret", "uid": ..., "field": ...}'))] = Field(default_factory=dict) + lun_size: Annotated[util.Size, Field(description="Size of the LUN created for each added node; 0 = none")] = 0 + allow_local_kms_credentials: Annotated[bool | None, Field(description=( + "Opt in (true) or out (false) of keeping the credentials in the cluster's local key store, " + "effectively unencrypted, when it has no external KMS (test environments); omitted, unchanged" + ))] = None + journal: Annotated[Literal["local", "lun"], Field(description=( + "Where each node's journal is kept: local (a disk or partition of its host) or lun " + "(a LUN of its own, created next to its data LUN; needs lun_size)"))] = "local" + journal_lun_size: Annotated[util.Size, Field(description=( + "journal lun: size of each node's journal LUN; 0 = 3% of lun_size, at least 2 GiB"))] = 0 + journal_options: Annotated[dict[str, Any], Field(description=( + 'journal lun: plugin options for the journal LUNs, applied over options, e.g. {"pool": "fast-pool"}' + ))] = Field(default_factory=dict) + + +class ArrayDTO(BaseModel): + id: str + plugin: str + endpoint: str + options: dict[str, Any] + lun_size: int + journal: str = "local" + journal_lun_size: int = 0 + journal_options: dict[str, Any] = Field(default_factory=dict) + has_credentials: bool + credential_ref: dict[str, Any] = Field(default_factory=dict) + capabilities: dict[str, Any] | None = None + + +@api.post('/', name='clusters:arrays:create', status_code=201, response_model=ArrayDTO) +def register(cluster: Cluster, parameters: ArrayParams) -> ArrayDTO: + """Register the cluster's storage array (or re-register it, e.g. to + rotate its credentials). The plugin is loaded and connected with the + credentials and its capabilities read before anything is saved.""" + try: + view = array_controller.register_array( + cluster.get_id(), parameters.plugin, parameters.endpoint, parameters.options, + parameters.credentials, parameters.lun_size, + credential_ref=parameters.credential_ref or None, journal=parameters.journal, + journal_lun_size=parameters.journal_lun_size, journal_options=parameters.journal_options or None, + allow_local_kms_credentials=parameters.allow_local_kms_credentials) + except ValueError as e: + raise HTTPException(400, str(e)) + return ArrayDTO(**view) + + +@api.get('/', name='clusters:arrays:list', response_model=list[ArrayDTO]) +def list_arrays(cluster: Cluster) -> list[ArrayDTO]: + return [ArrayDTO(**a) for a in array_controller.get_arrays(cluster.get_id())] + + +@api.get('/{array_id}', name='clusters:arrays:get', response_model=ArrayDTO) +def get(cluster: Cluster, array_id: str) -> ArrayDTO: + for a in array_controller.get_arrays(cluster.get_id()): + if a["id"] == array_id: + return ArrayDTO(**a) + raise HTTPException(404, f"array {array_id} not found") + + +@api.delete('/{array_id}', name='clusters:arrays:delete', status_code=204, + responses={204: {"content": None}}) +def delete(cluster: Cluster, array_id: str, force: bool = False) -> Response: + """Remove the registration and its credentials. Refused while LUNs + created through the array exist, unless ``force``.""" + try: + array_controller.delete_array(cluster.get_id(), array_id, force=force) + except KeyError as e: + raise HTTPException(404, str(e.args[0] if e.args else e)) + except ValueError as e: + raise HTTPException(409, str(e)) + return Response(status_code=204) diff --git a/simplyblock_web/api/v2/cluster/storage_node/device.py b/simplyblock_web/api/v2/cluster/storage_node/device.py index c833155afb..3adfed29c3 100644 --- a/simplyblock_web/api/v2/cluster/storage_node/device.py +++ b/simplyblock_web/api/v2/cluster/storage_node/device.py @@ -4,7 +4,6 @@ from uuid import UUID from fastapi import APIRouter, HTTPException, Request, Response -from pydantic import BaseModel from sse_starlette import EventSourceResponse from simplyblock_core.controllers import device_controller @@ -134,35 +133,6 @@ def replace( ) -class DeviceResizeDTO(BaseModel): - device_id: str - old_size: int - new_size: int - placement_size: int - resized: bool - uniform: bool - warning: str = "" - - -@instance_api.post('/resize', name='clusters:storage_nodes:devices:resize', - response_model=DeviceResizeDTO, - responses={409: {"description": "The device cannot be grown"}}) -def resize(cluster: Cluster, storage_node: StorageNode, device: Device) -> DeviceResizeDTO: - """Grow an lblk device online to the size its block device now has. - - The storage node's host re-reads the size of the block device (for - example, a LUN grown on its storage array) and the device takes it over - without interrupting I/O. Shrinking is refused. The placement weight of - the device stays at its original size, so the added capacity is only - fully usable once all data devices of the cluster have grown by the same - factor; `uniform` and `warning` report it. - """ - try: - return DeviceResizeDTO(**device_controller.resize_device(device.get_id())) - except ValueError as e: - raise HTTPException(409, str(e)) from e - - @instance_api.post('/restart', name='clusters:storage_nodes:devices:restart', status_code=204, responses={204: {"content": None}}) def restart(cluster: Cluster, storage_node: StorageNode, device: Device, force: bool = False) -> Response: if not device_controller.restart_device(device.get_id(), force): diff --git a/simplyblock_web/node_configure.py b/simplyblock_web/node_configure.py index a3b39000b0..0e20366062 100755 --- a/simplyblock_web/node_configure.py +++ b/simplyblock_web/node_configure.py @@ -220,6 +220,18 @@ def parse_arguments() -> argparse.Namespace: dest='blk_serials', required=False ) + parser.add_argument( + '--lblk-journal-only', + help="With --lblk on a multi-attach cluster whose array plugin creates each node's data LUN at add-node: the one selected whole disk is the node's journal and nothing else. It is never partitioned, and the node has no local data device; its data device is the array LUN. Needs exactly one disk, named with --blk-names or --blk-serials.", + action='store_true', + dest='lblk_journal_only', + ) + parser.add_argument( + '--lblk-array-only', + help="With --lblk on a multi-attach cluster whose array plugin creates each node's LUNs at add-node and keeps its journal on a LUN: the node uses no local device. Its journal LUN and data LUN are created on the array when the node is added. Not with --blk-names, --blk-names-exclude, --blk-serials, --lblk-journal-only or --lblk-journal-device.", + action='store_true', + dest='lblk_array_only', + ) parser.add_argument( '--lblk-journal-device', help='With --lblk and whole disks: dedicate each node\'s smallest disk to the ' @@ -290,6 +302,25 @@ def validate_arguments(args: argparse.Namespace) -> None: None, "--blk-names/--blk-names-exclude/--blk-serials require --lblk") if getattr(args, 'lblk_journal_device', False) and not lblk: raise argparse.ArgumentError(None, "--lblk-journal-device requires --lblk") + if getattr(args, 'lblk_journal_only', False): + if not lblk: + raise argparse.ArgumentError(None, "--lblk-journal-only requires --lblk") + if getattr(args, 'lblk_journal_device', False): + raise argparse.ArgumentError( + None, "--lblk-journal-only and --lblk-journal-device exclude each other") + if not (args.blk_names or args.blk_serials): + raise argparse.ArgumentError( + None, "--lblk-journal-only needs the journal disk or partition named with " + "--blk-names or --blk-serials") + if getattr(args, 'lblk_array_only', False): + if not lblk: + raise argparse.ArgumentError(None, "--lblk-array-only requires --lblk") + if (getattr(args, 'lblk_journal_only', False) or getattr(args, 'lblk_journal_device', False) + or blk_names or blk_names_exclude or blk_serials): + raise argparse.ArgumentError( + None, "--lblk-array-only selects no local device: not with --blk-names, " + "--blk-names-exclude, --blk-serials, --lblk-journal-only or " + "--lblk-journal-device") if use_lblk and (args.pci_allowed or args.pci_blocked or getattr(args, 'device_model', '') or getattr(args, 'size_range', '') @@ -367,6 +398,8 @@ def main() -> None: "names_exclude": [x.strip() for x in args.blk_names_exclude.split(',') if x.strip()] or None, "serials": [x.strip() for x in args.blk_serials.split(',') if x.strip()] or None, "journal_device": bool(getattr(args, 'lblk_journal_device', False)), + "journal_only": bool(getattr(args, 'lblk_journal_only', False)), + "array_only": bool(getattr(args, 'lblk_array_only', False)), # Read before the discard below: the disks of an existing node # with a dedicated journal device are never partitioned. "keep_serials": sorted(utils.previous_lblk_dedicated_journal_serials()), diff --git a/simplyblock_web/node_utils.py b/simplyblock_web/node_utils.py index 1abefb2559..6c2c06342d 100644 --- a/simplyblock_web/node_utils.py +++ b/simplyblock_web/node_utils.py @@ -317,7 +317,7 @@ def get_block_devices_info() -> list[dict]: logger.debug("function:get_block_devices_info start") out, err, rc = shell_utils.run_command( - "lsblk -J -b -o NAME,TYPE,SIZE,SERIAL,WWN,MOUNTPOINT,MODEL,ROTA,RO,VENDOR,PKNAME,PARTUUID") + "lsblk -J -b -o NAME,KNAME,TYPE,SIZE,SERIAL,WWN,MOUNTPOINT,MODEL,ROTA,RO,VENDOR,PKNAME,PARTUUID") if rc != 0: logger.error("Error running lsblk: %s", err) return [] @@ -350,8 +350,16 @@ def get_block_devices_info() -> list[dict]: serial = "SYN-" + hashlib.sha1(seed.encode()).hexdigest()[:16] synthetic = True numa_node = int(_read_sysfs(f"/sys/block/{name}/device/numa_node") or -1) + # The control plane picks the AIO bdev block size of a new device + # from these (utils.choose_aio_block_size): 4096 unless impossible. + logical_bs = _logical_block_size(name) + physical_bs = _physical_block_size(name) devices.append({ "name": name, + # Kernel name (dm-N for a device-mapper node, whose NAME is the + # map name) and TYPE: the control plane serves device-mapper + # nodes through bdev_uring (utils.lblk_io_engine_for). + "kname": dev.get("kname") or name, "device_path": f"/dev/{name}", "type": dev.get("type"), "size": int(dev.get("size") or 0), @@ -369,6 +377,8 @@ def get_block_devices_info() -> list[dict]: "by_id_path": by_id_path, "by_id_paths": by_id_paths, "numa_node": numa_node, + "logical_block_size": logical_bs, + "physical_block_size": physical_bs, }) for child in children: if child.get("type") != "part": @@ -385,6 +395,7 @@ def get_block_devices_info() -> list[dict]: part_synthetic = True devices.append({ "name": part_name, + "kname": child.get("kname") or part_name, "device_path": f"/dev/{part_name}", "type": "part", "size": int(child.get("size") or 0), @@ -405,11 +416,76 @@ def get_block_devices_info() -> list[dict]: "by_id_path": part_paths[0] if part_paths else "", "by_id_paths": part_paths, "numa_node": numa_node, + "logical_block_size": logical_bs, + "physical_block_size": physical_bs, + "start_bytes": _partition_start_bytes(name, part_name), }) + devices += _multipath_maps(data, devices, persistent) logger.debug("function:get_block_devices_info end") return devices +def _multipath_maps(lsblk: dict, devices: list[dict], persistent: dict) -> list[dict]: + """The dm-multipath maps of the host, one entry each. + + A SCSI LUN reached over several paths (FC, or iSCSI with several + sessions) appears once per path as an sd disk, and once as the map that + multipathd builds over those paths; lsblk lists the map as a child of + every path. The map is what is used: it survives the loss of a path. It + is reported with the identity of its paths (lsblk gives a map neither a + SERIAL nor a WWN), so a LUN is found under the same serial and WWN with + and without multipathing. Its paths stay in the inventory, each marked + with its ``multipath_map`` and held by it, so that nothing takes a path + on its own. The map is opened as /dev/mapper/; its kernel name + (dm-N) is what makes the control plane serve it through bdev_uring.""" + by_name = {d["name"]: d for d in devices if d.get("type") == "disk"} + maps: dict[str, dict] = {} + for dev in lsblk.get("blockdevices", []): + if dev.get("type") != "disk": + continue + for child in dev.get("children") or []: + if child.get("type") != "mpath": + continue + entry = maps.setdefault(child["name"], {"child": child, "paths": []}) + entry["paths"].append(dev["name"]) + out = [] + for map_name, m in sorted(maps.items()): + child, paths = m["child"], m["paths"] + kname = child.get("kname") or map_name + first = by_name.get(paths[0], {}) + for path in paths: + if path in by_name: + by_name[path]["multipath_map"] = map_name + links = persistent.get(kname, []) or persistent.get(map_name, []) + # The stable, host-independent link first: dm-uuid-mpath-. + links = sorted(links, key=lambda link: (0 if "/dm-uuid-mpath-" in link else 1, link)) + out.append({ + "name": map_name, + "kname": kname, + "device_path": f"/dev/mapper/{map_name}", + "type": "mpath", + "size": int(child.get("size") or first.get("size") or 0), + "serial": first.get("serial", ""), + "serial_synthetic": bool(first.get("serial_synthetic")), + "wwn": first.get("wwn", ""), + "model": first.get("model", ""), + "vendor": first.get("vendor", ""), + "rota": bool(first.get("rota")), + "ro": bool(child.get("ro")), + "has_partitions": any(c.get("type") == "part" for c in child.get("children") or []), + "mounted_in_subtree": _subtree_mounted(child), + "holders": _disk_holders(kname), + "is_root_disk": False, + "by_id_path": links[0] if links else "", + "by_id_paths": links, + "numa_node": first.get("numa_node", -1), + "logical_block_size": _logical_block_size(kname), + "physical_block_size": _physical_block_size(kname), + "paths": sorted(paths), + }) + return out + + SB_GPT_PARTITION_TYPECODE = "6527994e-2c5a-4eec-9613-8f5944074e8b" @@ -438,6 +514,134 @@ def simplyblock_data_signature(device_name: str) -> bool | None: return head.startswith(ALCEML_STORAGE_SIGNATURE) +def _normalize_wwn(value: str) -> str: + # lsblk reports a SCSI WWN as 0x, arrays as naa.; the same id. + v = (value or "").strip().lower() + return v.removeprefix("0x").removeprefix("naa.") + + +def _host_file(path: str) -> str: + """Contents of a host configuration file: the host's /etc as mounted into + the Kubernetes agent (/host/etc) first, then the local one (Docker agent + on the host).""" + for candidate in (f"/host{path}", path): + text = _read_sysfs(candidate) + if text: + return text + return "" + + +def get_san_initiators() -> dict: + """The host's SAN initiator identities, for registering it on a storage + array: ``iscsi_iqn`` (/etc/iscsi/initiatorname.iscsi), ``fc_wwpns`` (FC + HBA ports from /sys/class/fc_host), ``nvme_hostnqn`` (/etc/nvme/hostnqn) + and ``sdc_guid`` (PowerFlex SDC). Missing ones are left out. Read-only.""" + out: dict = {} + for line in _host_file("/etc/iscsi/initiatorname.iscsi").splitlines(): + line = line.strip() + if line.startswith("InitiatorName=") and not line.startswith("#"): + out["iscsi_iqn"] = line.split("=", 1)[1].strip() + break + wwpns = [] + base = "/sys/class/fc_host" + if os.path.isdir(base): + for host in sorted(os.listdir(base)): + name = _read_sysfs(f"{base}/{host}/port_name").lower() + if re.fullmatch(r"0x[0-9a-f]{16}", name) and int(name, 16): + wwpns.append(name.removeprefix("0x")) + if wwpns: + out["fc_wwpns"] = wwpns + nqn = _host_file("/etc/nvme/hostnqn").strip() + if nqn: + out["nvme_hostnqn"] = nqn + guid = "" + for drv_cfg in ("/opt/emc/scaleio/sdc/bin/drv_cfg", "/host/opt/emc/scaleio/sdc/bin/drv_cfg"): + if os.path.exists(drv_cfg): + guid, _, _ = shell_utils.run_command(f"{drv_cfg} --query_guid") + break + match = re.search(r"\b([0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12})\b", + guid or "") + if match: + out["sdc_guid"] = match.group(1) + return out + + +def scan_for_new_luns() -> dict: + """Make the kernel look for LUNs / namespaces newly presented to this host + by a storage array: every SCSI host (FC HBAs and iSCSI sessions alike) is + scanned (``- - -`` = all channels, targets, LUNs) and every NVMe + controller rescans its namespaces. Existing devices are not touched. + + The host must already have its SAN paths: iSCSI sessions logged in, FC + zoning done, NVMe-oF controllers connected. Returns ``{"scanned": [...]}``.""" + import glob as _glob + scanned: list[str] = [] + for path in sorted(_glob.glob("/sys/class/scsi_host/host*/scan")): + try: + _write_sysfs(path, "- - -") + scanned.append(path) + except OSError as e: + logger.warning("SCSI host scan %s failed: %s", path, e) + for path in sorted(_glob.glob("/sys/class/nvme/nvme*/rescan_controller")): + try: + _write_sysfs(path, "1") + scanned.append(path) + except OSError as e: + logger.warning("NVMe rescan %s failed: %s", path, e) + return {"scanned": scanned} + + +def _controller_dead(disk_name: str) -> bool: + """An NVMe namespace whose controller the kernel gave up on (state + ``dead``): no I/O can reach it or be queued on it.""" + return bool(disk_name) and _read_sysfs(f"/sys/block/{disk_name}/device/state") == "dead" + + +def find_lun_devices(wwn: str) -> list[dict]: + """The whole disks of this host that carry LUN ``wwn`` (multi-attach array + LUN mode): every lsblk disk whose WWN, or serial (the inventory falls back + to the WWN for it), matches, case-insensitive and with or without ``0x``. + + One dict per match: ``name``, ``device_path``, ``by_id_path``, + ``by_id_paths``, ``size`` (bytes), ``ro`` and ``has_alceml_signature`` -- + True when block 0 carries the alceml storage signature, False when it was + read and does not, None when it could not be read (a fenced or failed + path). Read-only: nothing is written to the device. Several matches mean + several host paths to the LUN; the caller chooses.""" + want = _normalize_wwn(wwn) + if not want: + return [] + found = [] + devices = get_block_devices_info() + matched = [d for d in devices + if d.get("type") in ("disk", "mpath") + and want in (_normalize_wwn(d.get("wwn", "")), _normalize_wwn(d.get("serial", "")))] + # A disk whose NVMe controller is dead holds no path to the LUN and no + # I/O for it: the kernel keeps such a 0-byte remnant when the volume was + # detached while the host booted (2026-11-21 FTT2 bed: the fenced node, + # back from a hard reset, kept reporting the detached LUN and stayed + # fenced for good). + matched = [d for d in matched if not _controller_dead(d.get("name", ""))] + # Several paths of one LUN under a multipath map: the map, not the paths. + if any(d.get("type") == "mpath" for d in matched): + matched = [d for d in matched if d.get("type") == "mpath"] + for dev in matched: + found.append({ + "name": dev["name"], + "device_path": dev["device_path"], + "by_id_path": dev.get("by_id_path", ""), + "by_id_paths": dev.get("by_id_paths", []), + "size": dev.get("size", 0), + "ro": dev.get("ro", False), + "type": dev.get("type", "disk"), + "kname": dev.get("kname", dev["name"]), + # /dev/ of a map is /dev/mapper/; /dev/ is dm-N. + "has_alceml_signature": simplyblock_data_signature( + dev.get("kname") or dev["name"] if dev.get("type") == "mpath" else dev["name"]), + }) + return found + + def _logical_block_size(disk_name: str) -> int: """Logical sector size of a disk, in bytes: the unit sgdisk counts in. sysfs reports partition start and size in 512-byte units regardless.""" @@ -448,6 +652,25 @@ def _logical_block_size(disk_name: str) -> int: return size if size > 0 else 512 +def _physical_block_size(disk_name: str) -> int: + """Physical sector size of a disk, in bytes (informational: a 512e + device reports 512 logical / 4096 physical).""" + try: + size = int(_read_sysfs(f"/sys/block/{disk_name}/queue/physical_block_size") or 0) + except ValueError: + size = 0 + return size if size > 0 else _logical_block_size(disk_name) + + +def _partition_start_bytes(disk_name: str, part_name: str) -> int | None: + """Byte offset of a partition on its disk; sysfs counts the start in + 512-byte units whatever the logical block size. None when unreadable.""" + try: + return int(_read_sysfs(f"/sys/block/{disk_name}/{part_name}/start")) * 512 + except ValueError: + return None + + def split_partition_partner(part: dict, devices: list[dict]) -> tuple[dict, dict] | None: """The ``(journal, data)`` pair a previous split_partition_for_journal made of a partition, when ``part`` is one of the two; otherwise None. @@ -784,7 +1007,9 @@ def rescan_block_device(device_path: str) -> dict: SCSI disks (FC, iSCSI, SAS) are rescanned through ``/sys/block//device/rescan``; NVMe namespaces through ``rescan_controller`` of every controller that reaches them, which also - covers the head device of native NVMe multipathing. Other devices, such + covers the head device of native NVMe multipathing. A dm-multipath map + (``dm-N`` with an ``mpath-`` uuid) has each of its path devices + rescanned, then ``multipathd resize map``. Other devices, such as virtio-blk, pick up a resize on their own, so their size is just read. Partitions are refused: growing one is a partition-table change, not a rescan. @@ -806,19 +1031,39 @@ def rescan_block_device(device_path: str) -> dict: raise ValueError(f"{name} is a partition, which cannot be resized online") rescanned: list[str] = [] - if _re.match(r"^nvme\d+n\d+$", name): - # A plain namespace links device -> its controller; the head device - # of native multipathing links device -> the subsystem, whose - # controllers are its nvme* children. - candidates = [f"{sys_dev}/device/rescan_controller"] - candidates += sorted(_glob.glob(f"{sys_dev}/device/nvme*/rescan_controller")) - for path in candidates: - if os.path.exists(path): - _write_sysfs(path, "1") - rescanned.append(path) - elif os.path.exists(f"{sys_dev}/device/rescan"): - _write_sysfs(f"{sys_dev}/device/rescan", "1") - rescanned.append(f"{sys_dev}/device/rescan") + + def _rescan_disk(disk: str, sys_disk: str) -> None: + if _re.match(r"^nvme\d+n\d+$", disk): + # A plain namespace links device -> its controller; the head + # device of native multipathing links device -> the subsystem, + # whose controllers are its nvme* children. + candidates = [f"{sys_disk}/device/rescan_controller"] + candidates += sorted(_glob.glob(f"{sys_disk}/device/nvme*/rescan_controller")) + for path in candidates: + if os.path.exists(path): + _write_sysfs(path, "1") + rescanned.append(path) + elif os.path.exists(f"{sys_disk}/device/rescan"): + _write_sysfs(f"{sys_disk}/device/rescan", "1") + rescanned.append(f"{sys_disk}/device/rescan") + + if _re.match(r"^dm-\d+$", name): + # A dm-multipath map (array LUN over several paths): every path + # device is rescanned first, then multipathd resizes the map, which + # only grows once all its paths report the new size. + slaves = sorted(os.listdir(f"{sys_dev}/slaves")) if os.path.isdir(f"{sys_dev}/slaves") else [] + for slave in slaves: + if _re.match(r"^[a-zA-Z0-9_\-]+$", slave): + _rescan_disk(slave, f"/sys/class/block/{slave}") + map_name = _read_sysfs(f"{sys_dev}/dm/name") + uuid = _read_sysfs(f"{sys_dev}/dm/uuid") + if map_name and uuid.startswith("mpath-") and _re.match(r"^[A-Za-z0-9_.\-]+$", map_name): + out, err, rc = shell_utils.run_command(f"multipathd resize map {map_name}") + if rc != 0 or "fail" in (out or "").lower(): + raise ValueError(f"multipathd resize map {map_name} failed: {err or out}") + rescanned.append(f"multipathd:{map_name}") + else: + _rescan_disk(name, sys_dev) sectors = _read_sysfs(f"{sys_dev}/size") if not sectors: diff --git a/tests/integration/kms/test_kmip_client.py b/tests/integration/kms/test_kmip_client.py index e37470362a..e817737659 100644 --- a/tests/integration/kms/test_kmip_client.py +++ b/tests/integration/kms/test_kmip_client.py @@ -114,3 +114,69 @@ def test_rekey_moves_keys_to_a_new_kek_and_path(kmip_client): def test_delete_key_encryption_key_of_an_unknown_name_is_a_no_op(kmip_client): kmip_client.delete_key_encryption_key(_unique("kek")) + + +# -- Opaque secrets (credentials for external systems) ---------------------- + +_CTX = "array/a1/https://pure01.example:443" + + +def test_wrap_secret_round_trip(kmip_client): + kek_name = _unique("kek") + kmip_client.ensure_key_encryption_key(kek_name) + try: + wrapped = kmip_client.wrap_secret( + kek_name, {"username": SecretStr("u"), "password": SecretStr("p")}, _CTX) + fields = kmip_client.unwrap_secret(kek_name, wrapped, _CTX) + assert {k: v.get_secret_value() for k, v in fields.items()} == { + "username": "u", "password": "p"} + with pytest.raises(KMSException, match="different context"): + kmip_client.unwrap_secret(kek_name, wrapped, "array/a1/https://evil.example:443") + finally: + kmip_client.delete_key_encryption_key(kek_name) + + +def test_ensure_key_encryption_key_keeps_existing_key(kmip_client): + kek_name = _unique("kek") + kmip_client.ensure_key_encryption_key(kek_name) + try: + wrapped = kmip_client.wrap_secret(kek_name, {"a": SecretStr("b")}, _CTX) + kmip_client.ensure_key_encryption_key(kek_name) + assert kmip_client.unwrap_secret(kek_name, wrapped, _CTX)["a"].get_secret_value() == "b" + finally: + kmip_client.delete_key_encryption_key(kek_name) + + +@pytest.fixture() +def kmip_client_no_fdb(pykmip_server): + """SecretData Get never touches FoundationDB.""" + client = KMIPClient( + pykmip_server.server_url, + pykmip_server.ca, pykmip_server.certificate, pykmip_server.key, + ) + with client: + yield client + + +def test_get_secret_data_returns_customer_secret(kmip_client_no_fdb): + from kmip.core import enums + from kmip.pie import objects + + value = b'{"username": "u", "password": "p"}' + uid = kmip_client_no_fdb.client.register( + objects.SecretData(value, enums.SecretDataType.PASSWORD)) + try: + assert kmip_client_no_fdb.get_secret_data(uid) == value + finally: + kmip_client_no_fdb.client.destroy(uid) + + +def test_get_secret_data_refuses_a_key_object(kmip_client_no_fdb): + from kmip.core import enums + + uid = kmip_client_no_fdb.client.create(enums.CryptographicAlgorithm.AES, 256) + try: + with pytest.raises(KMSException, match="not SecretData"): + kmip_client_no_fdb.get_secret_data(uid) + finally: + kmip_client_no_fdb.client.destroy(uid) diff --git a/tests/integration/test_cluster_suspend_recovery.py b/tests/integration/test_cluster_suspend_recovery.py index e235f29e72..1dc2be8ef1 100644 --- a/tests/integration/test_cluster_suspend_recovery.py +++ b/tests/integration/test_cluster_suspend_recovery.py @@ -143,8 +143,9 @@ def _run(self, nodes, cluster): # ---- baseline behavior preserved ------------------------------------ def test_all_online_active(self): - # Healthy 1x2 cluster: 4 online nodes, devices all online. - nodes = [_node(f"n{i}") for i in range(4)] + # Healthy 1x2 cluster: 5 online nodes (the n+2k node floor), devices + # all online. + nodes = [_node(f"n{i}") for i in range(5)] c = _cluster(distr_ndcs=1, distr_npcs=2) self.assertEqual(self._run(nodes, c), Cluster.STATUS_ACTIVE) @@ -217,14 +218,14 @@ def test_unreachable_node_counts_as_affected(self): def test_unreachable_without_data_plane_loss_not_counted(self): # The gate itself: mgmt-unreachable with the data plane still serving # (quorum says connected) must NOT suspend — an API blip or mgmt-NIC - # flap leaves clients unaffected. + # flap leaves clients unaffected. 5 ONLINE nodes keep the cluster at + # the resource floors (n+2k), so the verdict is the suspend rules'. from simplyblock_core.services import storage_node_monitor as mod nodes = [ _node("n0", status=StorageNode.STATUS_UNREACHABLE), _node("n1", status=StorageNode.STATUS_UNREACHABLE), _node("n2", status=StorageNode.STATUS_UNREACHABLE), - _node("n3", status=StorageNode.STATUS_ONLINE), - ] + ] + [_node(f"on{i}", status=StorageNode.STATUS_ONLINE) for i in range(5)] c = _cluster(distr_ndcs=1, distr_npcs=2) with patch.object(mod, "is_node_data_plane_disconnected_quorum", return_value=False): @@ -254,9 +255,11 @@ def test_removed_node_does_not_count_as_affected(self): _node("n1", status=StorageNode.STATUS_ONLINE), _node("n2", status=StorageNode.STATUS_ONLINE), _node("n3", status=StorageNode.STATUS_ONLINE), + _node("n4", status=StorageNode.STATUS_ONLINE), + _node("n5", status=StorageNode.STATUS_ONLINE), ] c = _cluster(distr_ndcs=1, distr_npcs=2) - # 0 affected, 3 online -> ACTIVE + # 0 affected, 5 online (the n+2k node floor) -> ACTIVE self.assertEqual(self._run(nodes, c), Cluster.STATUS_ACTIVE) @@ -303,20 +306,26 @@ def test_lone_down_stays_active(self): _node("on-1", status=StorageNode.STATUS_ONLINE, mgmt_ip="10.0.0.2"), _node("on-2", status=StorageNode.STATUS_ONLINE, mgmt_ip="10.0.0.3"), _node("on-3", status=StorageNode.STATUS_ONLINE, mgmt_ip="10.0.0.4"), + _node("on-4", status=StorageNode.STATUS_ONLINE, mgmt_ip="10.0.0.5"), + _node("on-5", status=StorageNode.STATUS_ONLINE, mgmt_ip="10.0.0.6"), ] + # 5 ONLINE besides the DOWN one: at the n+2k node floor. c = _cluster(distr_ndcs=1, distr_npcs=2) self.assertEqual(self._run(nodes, c), Cluster.STATUS_ACTIVE) def test_many_down_nodes_never_suspend(self): # Even several DOWN nodes never tip the cluster to SUSPENDED — they - # recover via port-unblock. 3 DOWN + 1 ONLINE on a 1x2 cluster -> ACTIVE. + # recover via port-unblock. 3 DOWN (> k) + 5 ONLINE (the n+2k node + # floor, so the floors do not degrade it) on a 1x2 cluster -> ACTIVE. nodes = [ _node(f"down-{i}", status=StorageNode.STATUS_DOWN, mgmt_ip=f"10.0.0.{i + 1}", n_online_devs=2, n_offline_devs=0) for i in range(3) ] - nodes.append( - _node("on-1", status=StorageNode.STATUS_ONLINE, mgmt_ip="10.0.0.9")) + nodes += [ + _node(f"on-{i}", status=StorageNode.STATUS_ONLINE, mgmt_ip=f"10.0.0.{i + 10}") + for i in range(5) + ] c = _cluster(distr_ndcs=1, distr_npcs=2) self.assertEqual(self._run(nodes, c), Cluster.STATUS_ACTIVE) diff --git a/tests/integration/test_dhchap_pool_level.py b/tests/integration/test_dhchap_pool_level.py index de022f12b2..9dc6a20183 100644 --- a/tests/integration/test_dhchap_pool_level.py +++ b/tests/integration/test_dhchap_pool_level.py @@ -546,7 +546,8 @@ class TestLvolInheritsDhchapFromPool: def test_lvol_allowed_hosts_set_from_pool(self, db, cluster): """When pool.dhchap=True, add_lvol_ha populates lvol.allowed_hosts from pool.""" node = _write_node(db) - secondary = _write_node(db, uuid="node-dhchap-2", devices=0) + # One data device: every online primary member needs >= 1 (floors). + secondary = _write_node(db, uuid="node-dhchap-2", devices=1) node.secondary_node_id = secondary.get_id() node.write_to_db(db.kv_store) pool = _write_pool(db, hosts=[HOST_A, HOST_B]) diff --git a/tests/integration/test_dual_ft_e2e.py b/tests/integration/test_dual_ft_e2e.py index d18eb5b386..ce62bc552d 100644 --- a/tests/integration/test_dual_ft_e2e.py +++ b/tests/integration/test_dual_ft_e2e.py @@ -673,9 +673,10 @@ def cluster_env(ensure_cluster, mock_rpc_servers): n.active_tcp = True n.active_rdma = False n.data_nics = [_make_nic("127.0.0.1")] - # Give each primary node 2 devices (need ndcs+npcs+1=4 total across cluster) + # Give each primary node 3 devices: 6 in all, at or above the 1+2 + # scheme's data-device floor (n+2k = 5, simplyblock_core.floors). devs = [] - for d in range(2): + for d in range(3): dev = NVMeDevice() dev.uuid = str(_uuid_mod.uuid4()) dev.cluster_id = cluster.uuid diff --git a/tests/integration/test_lblk_device_resize.py b/tests/integration/test_lblk_device_resize.py index 18cf41b5c1..4dc0bce163 100644 --- a/tests/integration/test_lblk_device_resize.py +++ b/tests/integration/test_lblk_device_resize.py @@ -3,19 +3,20 @@ What runs REAL here: the FDB persistence of the grown device (size and the frozen placement_size), the cluster-map build from the persisted records, -and device_controller.resize_device / grow_cluster_devices end to end. What +and device_controller.grow_cluster_devices end to end. What is faked: SPDK RPC, the node agent (rescan_block_device), and the event fan-out. Scenarios: - 1. A grown device persists its new size and keeps its placement size, and - the cluster map built from the stored records carries the old weight. - 2. A second growth keeps the original placement size. - 3. grow_cluster_devices grows every data device by the same factor and - reports the cluster as uniform; the journal device is left alone. + 1. A uniform growth persists the new sizes and keeps the placement sizes, + and the cluster map built from the stored records carries the old + weights; a second growth keeps the original placement size. + 2. Mixed growth (1.5 / 2.0) grows every data device by 1.5, records the + unused rest, and leaves the journal device alone. + 3. A shrunk block device is persisted and nothing grows. 4. A failed reopen leaves the stored size unchanged and the device unavailable. - 5. placement_size survives an FDB round-trip and defaults to 0. + 5. placement_size and backend_size survive an FDB round-trip. """ import uuid as uuid_mod @@ -106,21 +107,22 @@ def test_placement_size_round_trip(self, db): _seed_node(db, [dev]) read = db.get_storage_device_by_id(dev.get_id()) assert read.placement_size == 0 + assert read.backend_size == 0 assert read.weight_size() == 100 * _GIB - def test_grown_device_keeps_cluster_map_weight(self, db): + def test_uniform_growth_keeps_cluster_map_weight(self, db): _seed_cluster(db) a, b = _aio_device("S1", 0), _aio_device("S2", 1) node = _seed_node(db, [a, b]) - agent = _agent({a.by_id_path: 300 * _GIB}) + agent = _agent({a.by_id_path: 300 * _GIB, b.by_id_path: 300 * _GIB}) - res = _run(agent, MagicMock(), device_controller.resize_device, a.get_id()) + res = _run(agent, MagicMock(), device_controller.grow_cluster_devices, CLUSTER_ID) - assert res["resized"] is True - assert res["uniform"] is False + assert res["grown"] is True read = db.get_storage_device_by_id(a.get_id()) assert read.size == 300 * _GIB assert read.placement_size == 100 * _GIB + assert read.backend_size == 300 * _GIB fresh = db.get_storage_node_by_id(node.get_id()) cl_map = distr_controller.get_distr_cluster_map([fresh], fresh, "distr-1") @@ -133,9 +135,9 @@ def test_second_growth_keeps_original_placement(self, db): _seed_node(db, [a]) sizes = {a.by_id_path: 200 * _GIB} agent = _agent(sizes) - _run(agent, MagicMock(), device_controller.resize_device, a.get_id()) + _run(agent, MagicMock(), device_controller.grow_cluster_devices, CLUSTER_ID) sizes[a.by_id_path] = 400 * _GIB - _run(agent, MagicMock(), device_controller.resize_device, a.get_id()) + _run(agent, MagicMock(), device_controller.grow_cluster_devices, CLUSTER_ID) read = db.get_storage_device_by_id(a.get_id()) assert read.size == 400 * _GIB @@ -144,43 +146,43 @@ def test_second_growth_keeps_original_placement(self, db): class TestGrowCluster: - def test_uniform_growth_of_all_data_devices(self, db): + def test_mixed_growth_uses_the_smallest_factor(self, db): _seed_cluster(db) a, b = _aio_device("S1", 0), _aio_device("S2", 1) jm = _aio_device("J1", 2, size_gib=10, status=NVMeDevice.STATUS_JM) - _seed_node(db, [a, b, jm]) - agent = _agent({a.by_id_path: 200 * _GIB, b.by_id_path: 200 * _GIB}) + node = _seed_node(db, [a, b, jm]) + agent = _agent({a.by_id_path: 150 * _GIB, b.by_id_path: 200 * _GIB}) res = _run(agent, MagicMock(), device_controller.grow_cluster_devices, CLUSTER_ID) assert res["errors"] == {} - assert res["uniform"] is True - assert sorted(r["device_id"] for r in res["devices"]) == sorted([a.get_id(), b.get_id()]) + assert res["grown"] is True + assert res["factor"] == 1.5 + ra, rb = db.get_storage_device_by_id(a.get_id()), db.get_storage_device_by_id(b.get_id()) + assert (ra.size, rb.size) == (150 * _GIB, 150 * _GIB) + assert (ra.unused_size(), rb.unused_size()) == (0, 50 * _GIB) + assert (ra.placement_size, rb.placement_size) == (100 * _GIB, 100 * _GIB) assert db.get_storage_device_by_id(jm.get_id()).size == 10 * _GIB - assert res["rebalance"] is None - assert db.get_storage_device_by_id(a.get_id()).placement_size == 100 * _GIB - def test_uneven_growth_reweights_and_rebalances(self, db): + fresh = db.get_storage_node_by_id(node.get_id()) + cl_map = distr_controller.get_distr_cluster_map([fresh], fresh, "distr-1") + weights = [i["weight"] for i in cl_map["map_prob"][0]["items"]] + assert weights == [100, 100] + + def test_shrunk_device_is_persisted_and_nothing_grows(self, db): _seed_cluster(db) a, b = _aio_device("S1", 0), _aio_device("S2", 1) - node = _seed_node(db, [a, b]) - agent = _agent({a.by_id_path: 200 * _GIB, b.by_id_path: 300 * _GIB}) - - with patch.object(device_controller.tasks_controller, "add_device_mig_task_for_node", - return_value=True) as queue: - res = _run(agent, MagicMock(), device_controller.grow_cluster_devices, CLUSTER_ID) + _seed_node(db, [a, b]) + agent = _agent({a.by_id_path: 150 * _GIB, b.by_id_path: 80 * _GIB}) - assert res["uniform"] is False - assert res["rebalance"] == {"weights_updated": True, "started": True, "reason": ""} - queue.assert_called_once_with(node.get_id()) - assert db.get_storage_device_by_id(a.get_id()).placement_size == 200 * _GIB - assert db.get_storage_device_by_id(b.get_id()).placement_size == 300 * _GIB - assert db.get_cluster_by_id(CLUSTER_ID).reweight_pending is False + res = _run(agent, MagicMock(), device_controller.grow_cluster_devices, CLUSTER_ID) - fresh = db.get_storage_node_by_id(node.get_id()) - cl_map = distr_controller.get_distr_cluster_map([fresh], fresh, "distr-1") - weights = [i["weight"] for i in cl_map["map_prob"][0]["items"]] - assert weights == [200, 300] + assert res["grown"] is False + assert b.get_id() in res["errors"] + rb = db.get_storage_device_by_id(b.get_id()) + assert rb.size == 100 * _GIB + assert rb.is_shrunk() + assert db.get_storage_device_by_id(a.get_id()).size == 100 * _GIB def test_device_added_after_uniform_growth_is_weighted_on_the_same_scale(self, db): from simplyblock_core import storage_node_ops @@ -203,9 +205,10 @@ def test_failed_reopen_keeps_size_and_sets_unavailable(self, db): rpc = MagicMock() rpc.alceml_bdev_backend_resize_end.side_effect = RPCRemoteError("reopen", -7) - with pytest.raises(ValueError, match="unavailable"): - _run(agent, rpc, device_controller.resize_device, a.get_id()) + res = _run(agent, rpc, device_controller.grow_cluster_devices, CLUSTER_ID) + assert res["grown"] is False + assert "unavailable" in res["errors"][a.get_id()] read = db.get_storage_device_by_id(a.get_id()) assert read.size == 100 * _GIB assert read.placement_size == 0 diff --git a/tests/integration/test_shutdown_no_suspension.py b/tests/integration/test_shutdown_no_suspension.py index 0dfc659734..b1bc1a21cf 100644 --- a/tests/integration/test_shutdown_no_suspension.py +++ b/tests/integration/test_shutdown_no_suspension.py @@ -343,6 +343,12 @@ def bdev_nvme_detach_controller(self, name): detach_calls.append(name) return True peer.rpc_client = lambda **kwargs: _PeerRPC() + # A third node holding no controllers to the dying node: without it the + # cluster is a two-node one and the shutdown floor guard refuses + # (nodes online 1 < floor 2, simplyblock_core.floors). + bystander = _make_node("bystander", status=StorageNode.STATUS_ONLINE, + mgmt_ip="10.0.0.3") + bystander.rpc_client = lambda **kwargs: _PeerRPC() # dying-node SPDK kill + bind sink kill_calls = [] @@ -362,7 +368,7 @@ def bind_device_to_nvme(self, pci): def _get_node(nid): return snode if nid == snode.get_id() else peer db.get_storage_node_by_id.side_effect = _get_node - db.get_storage_nodes_by_cluster_id.return_value = [snode, peer] + db.get_storage_nodes_by_cluster_id.return_value = [snode, peer, bystander] db.get_job_tasks.return_value = [] db.kv_store = MagicMock() @@ -408,7 +414,7 @@ def _refresh(cluster_id): p.start() self.addCleanup(p.stop) fw_mock = MagicMock() # MUST NOT be called: shutdown fences nothing - route_fences([snode, peer], fw_mock) + route_fences([snode, peer, bystander], fw_mock) return { "snode": snode, "peer": peer, "db": db, diff --git a/tests/unit/storage_arrays/__init__.py b/tests/unit/storage_arrays/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/storage_arrays/conftest.py b/tests/unit/storage_arrays/conftest.py new file mode 100644 index 0000000000..44d8985189 --- /dev/null +++ b/tests/unit/storage_arrays/conftest.py @@ -0,0 +1,115 @@ +"""Shared fixtures: run ArrayClient against a vendor fake (in-process) or, +with ``SB_ARRAY_LIVE_CONFIG=``, against a real array/simulator. + +Live config file:: + + {"vendor": "netapp-ontap", "endpoint": "https://cluster1.lab:443", + "credential": {"username": "...", "password": "..."}, + "ca_bundle": "-----BEGIN CERTIFICATE-----...", "verify_tls": true, + "options": {"svm": "svm1"}, + "initiators": {"iscsi": [["iqn...a"], ["iqn...b"]]}} + +The live tier creates and deletes only ``sb-`` volumes and ``sbtest-`` hosts. +""" +from __future__ import annotations + +import importlib +import json +import os + +import pytest +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential +from simplyblock_core.storage_arrays.base import ArrayConfig, Initiator, Protocol +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.registry import ADAPTERS +from simplyblock_core.storage_arrays.testing import ContractSetup, FakeTransport + +VENDORS = sorted(ADAPTERS) + + +class StaticProvider: + """Stands in for CredentialProvider: returns ``credential`` and counts + invalidations. Tests flip ``credential`` to simulate rotation.""" + + def __init__(self, credential: Credential): + self.credential = credential + self.resolved: list[str] = [] + self.invalidated = 0 + + def resolve(self, ref, context): + self.resolved.append(context) + return self.credential + + def invalidate(self, ref=None): + self.invalidated += 1 + + +def fake_module(vendor: str): + tail = ADAPTERS[vendor].split(":")[0].split(".")[-1] + try: + return importlib.import_module(f"simplyblock_core.storage_arrays.testing.{tail}") + except ModuleNotFoundError as e: + if e.name and e.name.endswith(tail): + pytest.skip(f"no fake for {vendor} yet") + raise + + +class Harness: + def __init__(self, setup: ContractSetup, live: bool = False): + self.setup = setup + self.live = live + self.provider = StaticProvider(setup.credential) + self.client = ArrayClient( + setup.config, self.provider, # type: ignore[arg-type] + transport=None if live else FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}, + ) + + @property + def fake(self): + return self.setup.fake + + +def _live_setup(path: str) -> ContractSetup: + with open(path) as f: + cfg = json.load(f) + initiators = { + Protocol(proto): tuple( + tuple(Initiator(protocol=Protocol(proto), address=a) for a in group) + for group in groups) + for proto, groups in cfg["initiators"].items() + } + return ContractSetup( + fake=None, # type: ignore[arg-type] + config=ArrayConfig(id="live", vendor=cfg["vendor"], endpoint=cfg["endpoint"], + credential_ref={"kind": "vault-kv", "mount": "m", "path": "p"}, + ca_bundle=cfg.get("ca_bundle"), + verify_tls=cfg.get("verify_tls", True), + options=cfg.get("options", {})), + credential=Credential({k: SecretStr(v) for k, v in cfg["credential"].items()}), + bad_credential=Credential({k: SecretStr("wrong-" + v) + for k, v in cfg["credential"].items()}), + initiators=initiators, + create_foreign_volume=None, # type: ignore[arg-type] + has_lun_numbers=cfg.get("has_lun_numbers", True), + ) + + +def _params(): + live = os.environ.get("SB_ARRAY_LIVE_CONFIG") + if live: + return [pytest.param(("live", live), id="live")] + return [pytest.param(("fake", v), id=v) for v in VENDORS] + + +@pytest.fixture(params=_params()) +def harness(request) -> Harness: + kind, value = request.param + if kind == "live": + h = Harness(_live_setup(value), live=True) + else: + h = Harness(fake_module(value).contract_setup()) + yield h + h.client.close() diff --git a/tests/unit/storage_arrays/test_contract.py b/tests/unit/storage_arrays/test_contract.py new file mode 100644 index 0000000000..1e63e0fdc8 --- /dev/null +++ b/tests/unit/storage_arrays/test_contract.py @@ -0,0 +1,302 @@ +"""The contract every array adapter must meet, run against each vendor fake +(and, with SB_ARRAY_LIVE_CONFIG, against one real array or simulator).""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.base import ( + ArrayAuthError, + ArrayConflictError, + ArrayError, + ArrayRequestError, + HostSpec, + Protocol, +) +from simplyblock_core.storage_arrays.client import MANAGED_PREFIX, round_size + +GIB = 1 << 30 + + +def _uuid() -> str: + return str(uuid.uuid4()) + + +def _protocols(h): + return [p for p in h.setup.initiators if p in h.client.capabilities.protocols] + + +def _host(h, protocol, which=0, name=None): + return HostSpec(name=name or f"sbtest-{protocol.value}-{which}-{uuid.uuid4().hex[:6]}", + initiators=h.setup.initiators[protocol][which]) + + +@pytest.fixture() +def cleanup(harness): + volumes, hosts = [], [] + yield volumes, hosts + for vid in volumes: + try: + harness.client.delete_volume(vid) + except ArrayError: + pass + for host in hosts: + try: + harness.client.delete_host(host) + except ArrayError: + pass + + +def test_adapter_declares_itself(harness): + cls = harness.client.adapter_cls + assert cls.vendor == harness.client.vendor + assert cls.display_name + caps = cls.capabilities + assert caps.protocols + assert caps.size_granularity_bytes > 0 and caps.min_size_bytes > 0 + assert caps.max_name_len >= len(MANAGED_PREFIX) + 16 + assert set(_protocols(harness)), "fake provides no initiators for a supported protocol" + + +def test_get_info(harness): + info = harness.client.get_info() + assert info.vendor == harness.client.vendor + assert harness.provider.resolved[0].startswith("array/") + + +def test_create_get_find(harness, cleanup): + vol_uuid = _uuid() + vol = harness.client.create_volume(vol_uuid, 10 * GIB + 1) + cleanup[0].append(vol.id) + caps = harness.client.capabilities + assert vol.name == harness.client.volume_name(vol_uuid) + assert vol.name.startswith(MANAGED_PREFIX) + assert len(vol.name) <= caps.max_name_len + assert vol.size_bytes == round_size(10 * GIB + 1, caps.size_granularity_bytes, + caps.min_size_bytes) + assert vol.identifier and vol.identifier == vol.identifier.lower() + assert vol.wwn is None or vol.wwn.startswith(("naa.", "eui.")) + assert vol.nguid is None or vol.nguid.startswith("eui.") + + got = harness.client.get_volume(vol.id) + assert got == vol + assert harness.client.find_volume(vol_uuid) == vol + + +def test_missing_volume_is_none(harness): + assert harness.client.find_volume(_uuid()) is None + + +def test_get_volume_of_deleted_is_none(harness, cleanup): + vol = harness.client.create_volume(_uuid(), GIB) + assert harness.client.delete_volume(vol.id) is True + assert harness.client.get_volume(vol.id) is None + assert harness.client.delete_volume(vol.id) is False + + +def test_create_is_idempotent(harness, cleanup): + vol_uuid = _uuid() + first = harness.client.create_volume(vol_uuid, 4 * GIB) + cleanup[0].append(first.id) + again = harness.client.create_volume(vol_uuid, 4 * GIB) + assert again.id == first.id and again.identifier == first.identifier + + +def test_create_rerun_with_larger_size_grows(harness, cleanup): + vol_uuid = _uuid() + first = harness.client.create_volume(vol_uuid, 8 * GIB) + cleanup[0].append(first.id) + again = harness.client.create_volume(vol_uuid, 16 * GIB) + assert again.id == first.id and again.size_bytes >= 16 * GIB + + +def test_resize(harness, cleanup): + vol = harness.client.create_volume(_uuid(), 8 * GIB) + cleanup[0].append(vol.id) + grown = harness.client.resize_volume(vol.id, 24 * GIB, expected_identifier=vol.identifier) + assert grown.size_bytes >= 24 * GIB + assert grown.identifier == vol.identifier + assert harness.client.get_volume(vol.id).size_bytes == grown.size_bytes + # same size: no-op + assert harness.client.resize_volume(vol.id, grown.size_bytes) == grown + with pytest.raises(ArrayConflictError, match="shrink"): + harness.client.resize_volume(vol.id, 8 * GIB) + + +def test_host_and_mapping_lifecycle(harness, cleanup): + for protocol in _protocols(harness): + vol = harness.client.create_volume(_uuid(), GIB * 8) + cleanup[0].append(vol.id) + spec_a = _host(harness, protocol, 0) + spec_b = _host(harness, protocol, 1) + host_a = harness.client.ensure_host(spec_a) + cleanup[1].append(host_a) + assert harness.client.ensure_host(spec_a) == host_a, "ensure_host not idempotent" + host_b = harness.client.ensure_host(spec_b) + cleanup[1].append(host_b) + assert host_a != host_b + + m_a = harness.client.map_volume(vol.id, host_a, expected_identifier=vol.identifier) + assert m_a.host_id == host_a and m_a.volume_id == vol.id + if harness.setup.has_lun_numbers: + assert isinstance(m_a.lun, int) and m_a.lun >= 0 + assert harness.client.map_volume(vol.id, host_a) == m_a, "map not idempotent" + m_b = harness.client.map_volume(vol.id, host_b) # multi-attach + + hosts = {m.host_id for m in harness.client.list_mappings(vol.id)} + assert hosts == {host_a, host_b} + + harness.client.unmap_volume(vol.id, host_a) + harness.client.unmap_volume(vol.id, host_a) # idempotent + assert [m.host_id for m in harness.client.list_mappings(vol.id)] == [m_b.host_id] + + # delete unmaps the remaining host first + assert harness.client.delete_volume(vol.id, expected_identifier=vol.identifier) + cleanup[0].remove(vol.id) + assert harness.client.get_volume(vol.id) is None + + harness.client.delete_host(host_a) + harness.client.delete_host(host_a) # absent: no-op + harness.client.delete_host(host_b) + cleanup[1].clear() + + +def test_initiator_owned_by_another_host_conflicts(harness, cleanup): + protocol = _protocols(harness)[0] + first = _host(harness, protocol, 0) + cleanup[1].append(harness.client.ensure_host(first)) + other = HostSpec(name=f"sbtest-other-{uuid.uuid4().hex[:6]}", initiators=first.initiators) + with pytest.raises(ArrayConflictError): + cleanup[1].append(harness.client.ensure_host(other)) + + +def test_existing_host_gets_missing_initiators(harness, cleanup): + protocol = _protocols(harness)[0] + a, b = harness.setup.initiators[protocol] + name = f"sbtest-grow-{uuid.uuid4().hex[:6]}" + host = harness.client.ensure_host(HostSpec(name=name, initiators=a)) + cleanup[1].append(host) + limit = harness.client.capabilities.max_initiators_per_host + if limit is not None and len(a + b) > limit: + with pytest.raises(ArrayError, match="at most"): + harness.client.ensure_host(HostSpec(name=name, initiators=a + b)) + return + assert harness.client.ensure_host(HostSpec(name=name, initiators=a + b)) == host + # b's initiators now belong to that host: claiming them elsewhere conflicts + with pytest.raises(ArrayConflictError): + cleanup[1].append(harness.client.ensure_host( + HostSpec(name=f"sbtest-x-{uuid.uuid4().hex[:6]}", initiators=b))) + + +def test_wrong_identifier_is_refused(harness, cleanup): + vol = harness.client.create_volume(_uuid(), GIB) + cleanup[0].append(vol.id) + for action in ( + lambda: harness.client.delete_volume(vol.id, expected_identifier="naa.600000000000"), + lambda: harness.client.resize_volume(vol.id, 4 * GIB, expected_identifier="naa.6000"), + ): + with pytest.raises(ArrayConflictError, match="does not match"): + action() + assert harness.client.get_volume(vol.id) is not None + + +def test_volume_of_another_record_is_refused(harness, cleanup): + """A reused array id/WWN must not pass for the recorded volume: the + name derived from the recorded UUID pins it.""" + vol_uuid = _uuid() + vol = harness.client.create_volume(vol_uuid, GIB) + cleanup[0].append(vol.id) + with pytest.raises(ArrayConflictError, match="name does not match"): + harness.client.delete_volume(vol.id, vol.identifier, volume_uuid=_uuid()) + with pytest.raises(ArrayConflictError, match="name does not match"): + harness.client.resize_volume(vol.id, 4 * GIB, volume_uuid=_uuid()) + assert harness.client.delete_volume(vol.id, vol.identifier, volume_uuid=vol_uuid) + cleanup[0].remove(vol.id) + + +def test_foreign_volume_is_refused(harness): + if harness.live: + pytest.skip("live tier never touches foreign volumes") + vid = harness.setup.create_foreign_volume("customer-db01", 4 * GIB) + for action in (lambda: harness.client.delete_volume(vid), + lambda: harness.client.resize_volume(vid, 8 * GIB)): + with pytest.raises(ArrayConflictError, match="not created by simplyblock"): + action() + assert harness.client.get_volume(vid) is not None + + +def test_session_expiry_is_transparent(harness, cleanup): + if harness.live: + pytest.skip("cannot expire a real array's sessions") + harness.client.get_info() + harness.fake.expire_sessions() + vol = harness.client.create_volume(_uuid(), GIB) + cleanup[0].append(vol.id) + assert harness.client.get_volume(vol.id) == vol + + +def test_bad_credential_raises_auth_error_after_one_retry(harness): + harness.provider.credential = harness.setup.bad_credential + with pytest.raises(ArrayAuthError): + harness.client.get_info() + assert harness.provider.invalidated == 1 + assert len(harness.provider.resolved) == 2 + + +def test_rotated_credential_is_picked_up(harness): + if harness.live: + pytest.skip("needs control of the array's credential") + harness.client.get_info() + new = {k: "rotated-" + harness.setup.credential.get(k).get_secret_value() + for k in harness.setup.credential.field_names} + harness.fake.set_credential(**new) + harness.fake.expire_sessions() + from pydantic import SecretStr + + from simplyblock_core.credentials import Credential + harness.provider.credential = Credential({k: SecretStr(v) for k, v in new.items()}) + # The client still holds the old credential; the array rejects it, the + # client invalidates once, resolves the rotated secret and succeeds. + harness.client.get_info() + assert harness.provider.invalidated == 1 + + +def test_server_error_does_not_leak_credentials(harness): + if harness.live: + pytest.skip("fault injection needs the fake") + harness.client.get_info() + harness.fake.fail_next(500, {"error": "internal"}, times=50) + with pytest.raises(ArrayRequestError) as e: + harness.client.create_volume(_uuid(), GIB) + text = str(e.value) + for name in harness.setup.credential.field_names: + assert harness.setup.credential.get(name).get_secret_value() not in text + + +def test_unsupported_protocol_is_refused_before_calling_the_array(harness): + unsupported = [p for p in Protocol if p not in harness.client.capabilities.protocols] + if not unsupported: + pytest.skip("adapter supports every protocol") + from simplyblock_core.storage_arrays.base import Initiator + p = unsupported[0] + address = "21:00:00:24:ff:4c:1a:99" if p in (Protocol.FC, Protocol.NVME_FC) else "x" + with pytest.raises(ArrayError, match="does not support"): + harness.client.ensure_host(HostSpec(name="sbtest-unsup", + initiators=(Initiator(protocol=p, address=address),))) + + +def test_delete_host_with_mappings_is_refused(harness, cleanup): + """Deleting a host must never silently drop a live mapping.""" + protocol = _protocols(harness)[0] + vol = harness.client.create_volume(_uuid(), GIB) + cleanup[0].append(vol.id) + host = harness.client.ensure_host(_host(harness, protocol, 0)) + cleanup[1].append(host) + harness.client.map_volume(vol.id, host) + try: + harness.client.delete_host(host) + except ArrayConflictError: + pass # refused: the expected outcome on arrays that own host objects + # Either way (refused, or a no-op as for PowerFlex SDCs), the mapping survives. + assert [m.host_id for m in harness.client.list_mappings(vol.id)] == [host] diff --git a/tests/unit/storage_arrays/test_dell_powerflex.py b/tests/unit/storage_arrays/test_dell_powerflex.py new file mode 100644 index 0000000000..a8e5014ab7 --- /dev/null +++ b/tests/unit/storage_arrays/test_dell_powerflex.py @@ -0,0 +1,226 @@ +"""Dell PowerFlex specifics beyond the shared contract.""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.base import ( + ArrayConflictError, + ArrayError, + ArrayNotFoundError, + ArrayRequestError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeTransport, make_config +from simplyblock_core.storage_arrays.testing.dell_powerflex import ( + FakePowerFlex, + contract_setup, +) + +from .conftest import StaticProvider + +GIB = 1 << 30 + + +def _client(setup, **options): + config = setup.config + if options: + config = make_config("dell-powerflex", config.endpoint, + **{**config.options, **options}) + return ArrayClient(config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + + +def _sdc(setup, which=0) -> Initiator: + return setup.initiators[Protocol.SDC][which][0] + + +@pytest.fixture(params=[3, 4], ids=["v3-token", "v4-oauth"]) +def setup(request): + return contract_setup(generation=request.param) + + +def test_auto_auth_picks_scheme_by_api_version(setup): + # goscaleio / PyPowerFlex: /api/login on both generations, /api/version + # decides basic (3.x) vs Bearer (4.x); /rest/auth/login is not probed. + with _client(setup) as c: + info = c.get_info() + adapter = c._adapter + assert adapter is not None + bearer = adapter.session.headers.get("Authorization", "") + basic = adapter.session.http.auth + fake: FakePowerFlex = setup.fake + paths = [p for _, p in fake.requests] + assert paths[:2] == ["/api/login", "/api/version"] + assert "/rest/auth/login" not in paths + if fake.generation == 3: + assert "R3_6" in info.version and basic and not bearer + else: + assert "R4_5" in info.version and bearer.startswith("Bearer ") and not basic + # close() logged out + assert not fake.tokens and not fake.access_tokens + + +def test_pinned_oauth_on_4x_uses_access_token(): + setup = contract_setup(generation=4) + with _client(setup, auth="oauth") as c: + assert "R4_5" in c.get_info().version + assert setup.fake.access_tokens and not setup.fake.tokens + assert ("POST", "/rest/auth/login") in setup.fake.requests + assert ("POST", "/rest/auth/logout") in setup.fake.requests + assert not setup.fake.access_tokens + + +def test_add_mapped_sdc_body(setup): + with _client(setup) as c: + c.get_info() + adapter = c._adapter + assert adapter is not None + sent = [] + real = adapter.session.request + + def spy(method, path, **kw): + if path.endswith("/action/addMappedSdc"): + sent.append(kw["json"]) + return real(method, path, **kw) + adapter.session.request = spy # type: ignore[method-assign] + vol = adapter.create_volume("sb-body", 8 * GIB) + host = adapter.ensure_host(HostSpec(name="sbtest-body", initiators=(_sdc(setup),))) + adapter.map_volume(vol.id, host) + assert sent == [{"sdcId": host, "allowMultipleMappings": "true", + "accessMode": "ReadWrite"}] + + +def test_pinned_token_auth_skips_oauth_probe(): + setup = contract_setup(generation=3) + with _client(setup, auth="token") as c: + c.get_info() + assert ("POST", "/rest/auth/login") not in setup.fake.requests + + +def test_pinned_oauth_on_3x_fails(): + setup = contract_setup(generation=3) + with _client(setup, auth="oauth") as c, pytest.raises(ArrayNotFoundError): + c.get_info() + + +def test_bad_options_rejected(): + setup = contract_setup() + for opts in ({"auth": "kerberos"}, {"sdc_naming": "maybe"}, {"bogus": 1}): + with _client(setup, **opts) as c, pytest.raises(ArrayError): + c.get_info() + config = make_config("dell-powerflex", "https://pf.example") + with ArrayClient(config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake)) as c, \ + pytest.raises(ArrayError, match="storage_pool_id"): + c.get_info() + + +def test_sizes_round_to_8_gib(setup): + with _client(setup) as c: + vol = c.create_volume(str(uuid.uuid4()), 10 * GIB) + assert vol.size_bytes == 16 * GIB + raw = setup.fake.volumes[vol.id] + assert raw["sizeInKb"] == 16 * 1024 * 1024 + assert raw["volumeType"] == "ThinProvisioned" + grown = c.resize_volume(vol.id, 17 * GIB) + assert grown.size_bytes == 24 * GIB + assert vol.wwn is None and vol.nguid is None + assert vol.identifier == f"id.{vol.id}" + + +def test_not_found_500_is_none_but_other_500_raises(setup): + with _client(setup) as c: + assert c.get_volume("deadbeef00000001") is None + c.get_info() + setup.fake.fail_next(500, {"message": "MDM is busy", "httpStatusCode": 500, + "errorCode": 3}, path="Volume::") + with pytest.raises(ArrayRequestError, match="MDM is busy"): + c.get_volume("deadbeef00000001") + + +def test_duplicate_name_conflicts(setup): + setup.fake.add_volume("sb-taken", 8 * GIB) + with _client(setup) as c: + c.get_info() + adapter = c._adapter + assert adapter is not None + with pytest.raises(ArrayConflictError): + adapter.create_volume("sb-taken", 8 * GIB) + + +def test_ensure_host_finds_preregistered_sdc_and_claims_name(setup): + with _client(setup) as c: + host = c.ensure_host(HostSpec(name="worker-1", initiators=(_sdc(setup),))) + assert setup.fake.sdcs[host]["name"] == "worker-1" + # GUID match is case-insensitive + lower = Initiator(protocol=Protocol.SDC, address=_sdc(setup).address.lower()) + assert c.ensure_host(HostSpec(name="worker-1", initiators=(lower,))) == host + with pytest.raises(ArrayConflictError, match="worker-1"): + c.ensure_host(HostSpec(name="worker-2", initiators=(_sdc(setup),))) + # delete_host leaves the SDC alone + c.delete_host(host) + assert host in setup.fake.sdcs + + +def test_unknown_sdc_guid_is_not_found(setup): + ghost = Initiator(protocol=Protocol.SDC, address=str(uuid.uuid4())) + with _client(setup) as c, pytest.raises(ArrayNotFoundError, match="install the SDC"): + c.ensure_host(HostSpec(name="worker-9", initiators=(ghost,))) + + +def test_multi_sdc_host_is_refused_by_the_client(setup): + spec = HostSpec(name="worker-1", initiators=(_sdc(setup, 0), _sdc(setup, 1))) + with _client(setup) as c, pytest.raises(ArrayError, match="at most 1"): + c.ensure_host(spec) + + +def test_admin_named_sdc_and_ignore_mode(setup): + sid = setup.fake.add_sdc("ABCDEF01-0000-0000-0000-000000000001", name="esx-prod-7") + spec = HostSpec(name="worker-1", initiators=( + Initiator(protocol=Protocol.SDC, address="ABCDEF01-0000-0000-0000-000000000001"),)) + with _client(setup) as c, pytest.raises(ArrayConflictError): + c.ensure_host(spec) + with _client(setup, sdc_naming="ignore") as c: + assert c.ensure_host(spec) == sid + assert setup.fake.sdcs[sid]["name"] == "esx-prod-7" + + +def test_mapping_multi_sdc_and_null_mapped_info(setup): + with _client(setup) as c: + vol = c.create_volume(str(uuid.uuid4()), 8 * GIB) + assert setup.fake.volumes[vol.id]["mappedSdcInfo"] is None + assert c.list_mappings(vol.id) == [] + a = c.ensure_host(HostSpec(name="w-a", initiators=(_sdc(setup, 0),))) + b = c.ensure_host(HostSpec(name="w-b", initiators=(_sdc(setup, 1),))) + m = c.map_volume(vol.id, a) + assert m.lun is None + c.map_volume(vol.id, b) # needs allowMultipleMappings + assert {x.host_id for x in c.list_mappings(vol.id)} == {a, b} + c.unmap_volume(vol.id, a) + c.unmap_volume(vol.id, b) + assert setup.fake.volumes[vol.id]["mappedSdcInfo"] is None + assert c.delete_volume(vol.id) + + +def test_remove_mapped_volume_refused_by_array(setup): + with _client(setup) as c: + vol = c.create_volume(str(uuid.uuid4()), 8 * GIB) + host = c.ensure_host(HostSpec(name="w-a", initiators=(_sdc(setup, 0),))) + c.map_volume(vol.id, host) + adapter = c._adapter + assert adapter is not None + with pytest.raises(ArrayConflictError, match="mapped"): + adapter.delete_volume(vol.id) + + +def test_session_expiry_relogs_in(setup): + with _client(setup) as c: + c.get_info() + setup.fake.expire_sessions() + assert c.find_volume(str(uuid.uuid4())) is None diff --git a/tests/unit/storage_arrays/test_dell_powermax.py b/tests/unit/storage_arrays/test_dell_powermax.py new file mode 100644 index 0000000000..c9d7610d29 --- /dev/null +++ b/tests/unit/storage_arrays/test_dell_powermax.py @@ -0,0 +1,331 @@ +"""Dell PowerMax specifics beyond the shared contract: cylinder sizing, +async jobs, one storage group + masking view per volume/host, hex LUNs, +non-unique volume identifiers.""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.adapters.dell_powermax import ( + CYLINDER_BYTES, + DellPowerMaxAdapter, + powermax_nvme_initiator, +) +from simplyblock_core.storage_arrays.base import ( + AdapterContext, + ArrayConflictError, + ArrayError, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.http import RestSession +from simplyblock_core.storage_arrays.testing import FakeTransport +from simplyblock_core.storage_arrays.testing.dell_powermax import ( + SYMMETRIX_ID, + contract_setup, +) + +GIB = 1 << 30 + + +class _Provider: + def __init__(self, credential): + self.credential = credential + + def resolve(self, ref, context): + return self.credential + + def invalidate(self, ref=None): + pass + + +def _client(setup) -> ArrayClient: + return ArrayClient(setup.config, _Provider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + + +def _adapter(setup, **options) -> DellPowerMaxAdapter: + opts = dict(setup.config.options) + opts.update(options) + opts = {k: v for k, v in opts.items() if v is not None} + config = setup.config.model_copy(update={"options": opts}) + session = RestSession.for_config(config, transport=FakeTransport(setup.fake), + sleep=lambda s: None) + adapter = DellPowerMaxAdapter(AdapterContext(config, setup.credential, session)) + adapter.login() + return adapter + + +def _iscsi(name: str) -> HostSpec: + return HostSpec(name=name, initiators=( + Initiator(protocol=Protocol.ISCSI, address=f"iqn.2026-01.io.sb:{name}"),)) + + +@pytest.fixture() +def setup(): + return contract_setup() + + +def test_cylinder_granularity(setup): + client = _client(setup) + vol = client.create_volume(str(uuid.uuid4()), 10 * GIB) + assert CYLINDER_BYTES == 1_966_080 + assert vol.size_bytes % CYLINDER_BYTES == 0 + assert vol.size_bytes == 5462 * CYLINDER_BYTES # ceil(10 GiB / 1920 KiB) + assert setup.fake.volumes[vol.id]["cap_cyl"] == 5462 + + +def test_one_storage_group_per_volume(setup): + client = _client(setup) + vol = client.create_volume(str(uuid.uuid4()), GIB) + sg = setup.fake.storage_groups[vol.name] + assert sg["volumes"] == [vol.id] and sg["srp"] == "SRP_1" and sg["slo"] == "NONE" + assert vol.wwn == "naa." + setup.fake.volumes[vol.id]["effective_wwn"].lower() + assert client.delete_volume(vol.id) + assert vol.name not in setup.fake.storage_groups + assert setup.fake.volumes == {} + + +def test_service_level_is_applied(setup): + adapter = _adapter(setup, service_level="Diamond") + vol = adapter.create_volume("sb-diamond", GIB) + assert setup.fake.storage_groups[vol.name]["slo"] == "Diamond" + + +def test_async_jobs_are_polled(setup): + adapter = _adapter(setup, async_jobs=True) + vol = adapter.create_volume("sb-async", 2 * GIB) + vol = adapter.resize_volume(vol.id, 4 * GIB) + assert vol.size_bytes >= 4 * GIB + host = adapter.ensure_host(_iscsi("sbtest-async")) + m = adapter.map_volume(vol.id, host) + assert m.lun == 1 + adapter.unmap_volume(vol.id, host) + adapter.delete_volume(vol.id) + jobs = setup.fake.jobs.values() + assert len(jobs) >= 5 and all(j["status"] == "SUCCEEDED" for j in jobs) + assert all(j["_polls"] == 2 for j in jobs) + assert any("/system/job/" in path for _, path in setup.fake.requests) + + +def test_failed_job_is_classified(setup): + adapter = _adapter(setup, async_jobs=True) + vol = adapter.create_volume("sb-jobfail", 4 * GIB) + with pytest.raises(ArrayRequestError, match="FAILED.*larger"): + adapter.resize_volume(vol.id, 2 * GIB) + adapter.ensure_host(_iscsi("sbtest-j1")) + with pytest.raises(ArrayConflictError, match="already in use"): + adapter.ensure_host(HostSpec(name="sbtest-j2", + initiators=_iscsi("sbtest-j1").initiators)) + + +def test_job_timeout(setup): + adapter = _adapter(setup, async_jobs=True, job_timeout_s=2) + with pytest.raises(ArrayRequestError, match="did not finish"): + adapter.create_volume("sb-slow", GIB) + + +def test_hex_lun_addresses(setup): + setup.fake.first_lun = 0x1A + adapter = _adapter(setup) + vol = adapter.create_volume("sb-hexlun", GIB) + host = adapter.ensure_host(_iscsi("sbtest-hex")) + assert adapter.map_volume(vol.id, host).lun == 26 + assert adapter.list_mappings(vol.id)[0].lun == 26 + + +def test_port_group_by_protocol(setup): + adapter = _adapter(setup) + vol = adapter.create_volume("sb-pg", GIB) + fc = adapter.ensure_host(HostSpec(name="sbtest-fc", initiators=( + Initiator(protocol=Protocol.FC, address="21:00:00:24:FF:4C:1A:2B"),))) + assert setup.fake.hosts[fc]["initiator"] == ["21000024ff4c1a2b"] + adapter.map_volume(vol.id, fc) + view = setup.fake.views[DellPowerMaxAdapter.view_name(vol.name, fc)] + assert view["pg"] == "sb_pg_fc" + + +def test_single_port_group_name(setup): + adapter = _adapter(setup, port_group="sb_pg_iscsi") + vol = adapter.create_volume("sb-pg1", GIB) + host = adapter.ensure_host(_iscsi("sbtest-pg1")) + adapter.map_volume(vol.id, host) + assert setup.fake.views[f"{vol.name}_{host}"]["pg"] == "sb_pg_iscsi" + + +def test_map_without_port_group_is_refused(setup): + adapter = _adapter(setup, port_group=None) + vol = adapter.create_volume("sb-nopg", GIB) + host = adapter.ensure_host(_iscsi("sbtest-nopg")) + with pytest.raises(ArrayError, match="port_group"): + adapter.map_volume(vol.id, host) + with pytest.raises(ArrayNotFoundError): + adapter.map_volume(vol.id, "sbtest-missing") + + +def test_long_host_name_gets_hashed_view_name(setup): + adapter = _adapter(setup) + vol = adapter.create_volume("sb-" + "a" * 24, GIB) + name = "sbtest-" + "h" * 57 + host = adapter.ensure_host(_iscsi(name)) + adapter.map_volume(vol.id, host) + (view,) = setup.fake.views + assert len(view) <= 64 and view.startswith(vol.name + "_") + assert adapter.map_volume(vol.id, host).host_id == host # idempotent + + +def test_duplicate_identifiers(setup): + adapter = _adapter(setup) + vol = adapter.create_volume("sb-dup", GIB) + setup.fake.add_volume("sb-dup", GIB) # same identifier, outside our SG + assert adapter.find_volume("sb-dup") == vol + setup.fake.add_volume("sb-dup2", GIB) + setup.fake.add_volume("sb-dup2", GIB) + with pytest.raises(ArrayConflictError, match="2 PowerMax devices"): + adapter.find_volume("sb-dup2") + + +def test_leftover_empty_storage_group_is_reused(setup): + adapter = _adapter(setup) + setup.fake.storage_groups["sb-left"] = setup.fake._sg_record("sb-left", "SRP_1", None) + vol = adapter.create_volume("sb-left", GIB) + assert setup.fake.storage_groups["sb-left"]["volumes"] == [vol.id] + with pytest.raises(ArrayConflictError, match="already exists"): + adapter.create_volume("sb-left", GIB) + + +def test_shared_storage_group_mapping_is_not_unmapped(setup): + adapter = _adapter(setup) + vol = adapter.create_volume("sb-shared", GIB) + host = adapter.ensure_host(_iscsi("sbtest-shared")) + fake = setup.fake + fake.storage_groups["customer_sg"] = fake._sg_record("customer_sg", "SRP_1", None) + fake.storage_groups["customer_sg"]["volumes"].append(vol.id) + fake.volumes[vol.id]["storageGroupId"].append("customer_sg") + adapter._call("POST", f"{adapter._slo}/maskingview", json={ + "maskingViewId": "customer_mv", + "hostOrHostGroupSelection": {"useExistingHostParam": {"hostId": host}}, + "portGroupSelection": {"useExistingPortGroupParam": {"portGroupId": "sb_pg_iscsi"}}, + "storageGroupSelection": {"useExistingStorageGroupParam": { + "storageGroupId": "customer_sg"}}}) + assert [m.host_id for m in adapter.list_mappings(vol.id)] == [host] + with pytest.raises(ArrayConflictError, match="shared storage group"): + adapter.unmap_volume(vol.id, host) + + +def test_errors_and_lookups(setup): + adapter = _adapter(setup) + assert adapter.get_volume("0FFFF") is None + assert adapter.list_mappings("0FFFF") == [] + adapter.unmap_volume("0FFFF", "nobody") + adapter.delete_host("nobody") + with pytest.raises(ArrayNotFoundError): + adapter.delete_volume("0FFFF") + + +def test_get_info_and_version_check(setup): + info = _adapter(setup).get_info() + assert info.model == "PowerMax_8500" and info.serial == SYMMETRIX_ID + assert "6079" in info.version + with pytest.raises(ArrayUnsupportedError, match="92"): + _adapter(setup, api_version="92").get_info() + + +def test_unknown_symmetrix(setup): + with pytest.raises(ArrayNotFoundError): + _adapter(setup, symmetrix_id="000197900999").get_info() + + +def test_symmetrix_id_is_required(setup): + config = setup.config.model_copy(update={"options": {}}) + session = RestSession.for_config(config, transport=FakeTransport(setup.fake)) + with pytest.raises(ArrayError, match="symmetrix_id"): + DellPowerMaxAdapter(AdapterContext(config, setup.credential, session)) + + +def test_basic_auth_on_every_request(setup): + adapter = _adapter(setup) + assert adapter.session.http.auth == ("smc", "pmax-pass") + adapter.close() + assert adapter.session.http.auth is None + + +def test_nvme_initiator_ids_carry_the_host_id(setup): + # csi-powermax makeNVMeInitiatorIDs: ":" + nqn = "nqn.2014-08.org.nvmexpress:uuid:c32abcdf-35f9-4800-88ad-396225c90b70" + assert powermax_nvme_initiator(nqn) == nqn + ":C32ABCDF35F9480088AD396225C90B70" + explicit = "nqn.2026-01.io.sb:node1:c32abcdf35f9480088ad396225c90b70" + assert powermax_nvme_initiator(explicit) == ( + "nqn.2026-01.io.sb:node1:C32ABCDF35F9480088AD396225C90B70") + with pytest.raises(ArrayUnsupportedError, match="host ID"): + powermax_nvme_initiator("nqn.2026-01.io.sb:node1") + + adapter = _adapter(setup) + spec = HostSpec(name="sbtest-nvme", initiators=( + Initiator(protocol=Protocol.NVME_TCP, address=nqn),)) + host = adapter.ensure_host(spec) + assert setup.fake.hosts[host]["initiator"] == [nqn + ":C32ABCDF35F9480088AD396225C90B70"] + assert adapter.ensure_host(spec) == host # already present: no re-add + vol = adapter.create_volume("sb-nvme", GIB) + adapter.map_volume(vol.id, host) + assert setup.fake.views[f"{vol.name}_{host}"]["pg"] == "sb_pg_nvme" + with pytest.raises(ArrayUnsupportedError): + adapter.ensure_host(HostSpec(name="sbtest-nvme2", initiators=( + Initiator(protocol=Protocol.NVME_TCP, address="nqn.2026-01.io.sb:x"),))) + + +def test_storage_group_payload_names_workload_none(setup): + adapter = _adapter(setup) + sent = [] + real = adapter.session.request + + def spy(method, path, **kw): + if method == "POST" and path.endswith("/storagegroup"): + sent.append(kw["json"]) + return real(method, path, **kw) + adapter.session.request = spy # type: ignore[method-assign] + adapter.create_volume("sb-wl", GIB) + (param,) = sent[0]["sloBasedStorageGroupParam"] + assert param["sloId"] == "None" and param["workloadSelection"] == "None" + + +@pytest.mark.parametrize("state", ["VALIDATE_FAILED", "UNSCHEDULED"]) +def test_any_other_job_state_is_a_failure(setup, state): + setup.fake.job_failure_status = state + adapter = _adapter(setup, async_jobs=True) + vol = adapter.create_volume("sb-jobstate", 4 * GIB) + with pytest.raises(ArrayRequestError, match=f"{state}.*larger"): + adapter.resize_volume(vol.id, 2 * GIB) + + +def test_vendor_not_found_and_conflict_wording(setup): + adapter = _adapter(setup) + vol = adapter.create_volume("sb-words", GIB) + host = adapter.ensure_host(_iscsi("sbtest-words")) + # "cannot be found" (404 and 500) and "Could not find volume" are missing + assert adapter._call("GET", f"{adapter._slo}/host/nobody", missing_ok=True) is None + with pytest.raises(ArrayNotFoundError, match="cannot be found"): + adapter._call("POST", f"{adapter._slo}/maskingview", json={ + "maskingViewId": "sb_mv_nopg", + "hostOrHostGroupSelection": {"useExistingHostParam": {"hostId": host}}, + "portGroupSelection": {"useExistingPortGroupParam": {"portGroupId": "nopg"}}, + "storageGroupSelection": {"useExistingStorageGroupParam": { + "storageGroupId": vol.name}}}) + with pytest.raises(ArrayNotFoundError, match="Could not find volume"): + adapter.resize_volume("0FFFF", GIB) + # removing a device its storage group lacks: "does not contain" -> missing + other = setup.fake.add_volume("sb-other", GIB) + assert adapter._call("PUT", f"{adapter._slo}/storagegroup/{vol.name}", json={ + "editStorageGroupActionParam": {"removeVolumeParam": {"volumeId": [other]}}}, + missing_ok=True) is None + # 409 "already exists" is a conflict + with pytest.raises(ArrayConflictError, match="already exists"): + adapter._call("POST", f"{adapter._slo}/host", + json={"hostId": host, "initiatorId": []}) diff --git a/tests/unit/storage_arrays/test_dell_powerstore.py b/tests/unit/storage_arrays/test_dell_powerstore.py new file mode 100644 index 0000000000..be12010226 --- /dev/null +++ b/tests/unit/storage_arrays/test_dell_powerstore.py @@ -0,0 +1,248 @@ +"""Dell PowerStore specifics beyond the shared contract: CSRF token and +session cookie, PostgREST paging, 8 KiB granularity, 422 classification.""" +from __future__ import annotations + +import uuid + +import pytest +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential +from simplyblock_core.storage_arrays.adapters.dell_powerstore import ( + TOKEN_HEADER, + DellPowerStoreAdapter, + powerstore_nguid, + powerstore_wwn, +) +from simplyblock_core.storage_arrays.base import ( + AdapterContext, + ArrayConflictError, + ArrayError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.http import RestSession +from simplyblock_core.storage_arrays.testing import FakeTransport, make_config +from simplyblock_core.storage_arrays.testing.dell_powerstore import contract_setup + +GIB = 1 << 30 + + +class _Provider: + def __init__(self, credential): + self.credential = credential + + def resolve(self, ref, context): + return self.credential + + def invalidate(self, ref=None): + pass + + +@pytest.fixture() +def setup(): + return contract_setup() + + +@pytest.fixture() +def client(setup): + c = ArrayClient(setup.config, _Provider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + yield c + c.close() + + +def _adapter(setup, **options) -> DellPowerStoreAdapter: + config = setup.config.model_copy(update={"options": options}) + session = RestSession.for_config(config, transport=FakeTransport(setup.fake)) + adapter = DellPowerStoreAdapter(AdapterContext(config, setup.credential, session)) + adapter.login() + return adapter + + +def _iscsi(name: str) -> HostSpec: + return HostSpec(name=name, initiators=( + Initiator(protocol=Protocol.ISCSI, address=f"iqn.2026-01.io.sb:{name}"),)) + + +def test_login_keeps_cookie_and_csrf_token_not_basic_auth(setup): + adapter = _adapter(setup) + assert adapter.session.http.auth is None + assert adapter.session.headers[TOKEN_HEADER] in setup.fake.sessions.values() + assert "auth_cookie" in adapter.session.http.cookies + vol = adapter.create_volume("sb-csrf", GIB) + assert vol.size_bytes == GIB + + +def test_mutation_without_csrf_token_is_forbidden(setup): + adapter = _adapter(setup) + adapter.session.headers.pop(TOKEN_HEADER) + assert adapter.find_volume("sb-none") is None # GET needs only the cookie + with pytest.raises(ArrayError, match="403"): + adapter.create_volume("sb-nocsrf", GIB) + + +def test_close_logs_out(setup): + adapter = _adapter(setup) + assert len(setup.fake.sessions) == 1 + adapter.close() + assert setup.fake.sessions == {} + assert ("POST", "/api/rest/logout") in setup.fake.requests + + +def test_size_is_rounded_to_8k(client, setup): + vol = client.create_volume(str(uuid.uuid4()), GIB + 1) + assert vol.size_bytes == GIB + 8192 + assert setup.fake.volumes[vol.id]["size"] % 8192 == 0 + + +def test_fake_rejects_unaligned_size(setup): + adapter = _adapter(setup) + with pytest.raises(ArrayError, match="multiple of 8192"): + adapter.create_volume("sb-odd", GIB + 512) + + +def test_identity_format(client): + vol = client.create_volume(str(uuid.uuid4()), GIB) + assert vol.wwn and vol.wwn.startswith("naa.68ccf098") and vol.wwn == vol.wwn.lower() + assert vol.nguid and vol.nguid.startswith("eui.") and len(vol.nguid) == 4 + 32 + assert powerstore_wwn("NAA.68CCF098ABC") == "naa.68ccf098abc" + assert powerstore_nguid("nguid.ABCD") == "eui.abcd" + assert powerstore_wwn(None) is None and powerstore_nguid("") is None + + +def test_select_and_filter_are_sent(client, setup): + client.find_volume(str(uuid.uuid4())) + adapter = client._adapter + assert adapter is not None + # unknown select fields are refused by the array (400), not silently ignored + with pytest.raises(ArrayError, match="400"): + adapter._list("/volume", "id,bogus") + + +def test_paged_host_lookup(setup): + setup.fake.page_limit = 2 + adapter = _adapter(setup) + ids = [adapter.ensure_host(_iscsi(f"sbtest-page-{i}")) for i in range(5)] + assert len(set(ids)) == 5 + # the last host sits on the third page of an unfiltered listing + assert len(adapter._list("/host", "id,name")) == 5 + assert adapter.ensure_host(_iscsi("sbtest-page-4")) == ids[4] + + +def test_fc_initiators_are_colon_separated_and_deduplicated(setup): + adapter = _adapter(setup) + spec = HostSpec(name="sbtest-fc", initiators=( + Initiator(protocol=Protocol.FC, address="0x21000024FF4C1A2B"),)) + host_id = adapter.ensure_host(spec) + ports = setup.fake.hosts[host_id]["host_initiators"] + assert [(p["port_name"], p["port_type"]) for p in ports] == [ + ("21:00:00:24:ff:4c:1a:2b", "FC")] + grown = HostSpec(name="sbtest-fc", initiators=spec.initiators + ( + Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),)) + assert adapter.ensure_host(grown) == host_id + assert adapter.ensure_host(grown) == host_id + assert len(setup.fake.hosts[host_id]["host_initiators"]) == 2 + + +def test_nvme_host_uses_nvme_port_type(setup): + adapter = _adapter(setup) + spec = HostSpec(name="sbtest-nvme", initiators=( + Initiator(protocol=Protocol.NVME_TCP, address="nqn.2014-08.org.nvmexpress:uuid:x"),)) + host_id = adapter.ensure_host(spec) + assert setup.fake.hosts[host_id]["host_initiators"][0]["port_type"] == "NVMe" + assert setup.fake.hosts[host_id]["os_type"] == "Linux" + + +def test_initiator_conflict_is_classified(setup): + adapter = _adapter(setup) + adapter.ensure_host(_iscsi("sbtest-one")) + other = HostSpec(name="sbtest-two", initiators=_iscsi("sbtest-one").initiators) + with pytest.raises(ArrayConflictError, match="already registered"): + adapter.ensure_host(other) + + +def test_delete_attached_volume_is_conflict(setup): + adapter = _adapter(setup) + vol = adapter.create_volume("sb-attached", GIB) + host = adapter.ensure_host(_iscsi("sbtest-att")) + m = adapter.map_volume(vol.id, host) + assert m.lun == 1 + with pytest.raises(ArrayConflictError): + adapter.delete_volume(vol.id) + with pytest.raises(ArrayConflictError): + adapter.delete_host(host) + adapter.unmap_volume(vol.id, host) + adapter.delete_volume(vol.id) + assert adapter.get_volume(vol.id) is None + + +def test_luns_are_per_host(setup): + adapter = _adapter(setup) + host = adapter.ensure_host(_iscsi("sbtest-luns")) + luns = [adapter.map_volume(adapter.create_volume(f"sb-lun{i}", GIB).id, host).lun + for i in range(3)] + assert luns == [1, 2, 3] + + +def test_appliance_option_and_unknown_option(setup): + adapter = _adapter(setup, appliance_id="A2") + vol = adapter.create_volume("sb-a2", GIB) + assert setup.fake.volumes[vol.id]["appliance_id"] == "A2" + with pytest.raises(ArrayError, match="Unknown"): + _adapter(setup, pool="x") + + +def test_get_info(client): + info = client.get_info() + assert info.model == "PowerStore 1200T" + assert info.version == "PowerStoreOS 4.0.0.0" + assert info.serial == "PS4a7b5c3e9f01" + + +def test_api_token_credential_is_refused(): + setup = contract_setup() + config = make_config("dell-powerstore", "https://ps.example") + session = RestSession.for_config(config, transport=FakeTransport(setup.fake)) + with pytest.raises(ArrayError, match="username"): + DellPowerStoreAdapter(AdapterContext( + config, Credential({"api_token": SecretStr("t")}), session)) + + +def test_nvme_mapping_reports_volume_nsid(setup): + # gopowerstore volume_types.go: the NVMe namespace id is the volume's nsid + setup.fake.next_nsid = 42 + adapter = _adapter(setup) + vol = adapter.create_volume("sb-nsid", GIB) + nvme = adapter.ensure_host(HostSpec(name="sbtest-nsid", initiators=( + Initiator(protocol=Protocol.NVME_TCP, address="nqn.2014-08.org.nvmexpress:uuid:n"),))) + scsi = adapter.ensure_host(_iscsi("sbtest-nsid-scsi")) + assert adapter.map_volume(vol.id, nvme).lun == 42 + assert adapter.map_volume(vol.id, scsi).lun == 1 + assert {m.host_id: m.lun for m in adapter.list_mappings(vol.id)} == {nvme: 42, scsi: 1} + + +def test_detach_race_with_not_attached_is_idempotent(setup): + adapter = _adapter(setup) + vol = adapter.create_volume("sb-race", GIB) + host = adapter.ensure_host(_iscsi("sbtest-race")) + adapter.map_volume(vol.id, host) + # Detached elsewhere between unmap_volume's mapping check and its detach: + # the array answers 400 "Host is not attached to volume", read as missing. + listed = adapter._mappings(vol.id) + setup.fake.mappings.clear() + adapter._mappings = lambda vid: listed # type: ignore[method-assign] + adapter.unmap_volume(vol.id, host) + assert ("POST", f"/api/rest/volume/{vol.id}/detach") in setup.fake.requests + + +@pytest.mark.parametrize("status", [422, 500]) +def test_duplicate_volume_name_is_conflict(setup, status): + setup.fake.name_conflict_status = status + adapter = _adapter(setup) + adapter.create_volume("sb-dup", GIB) + with pytest.raises(ArrayConflictError, match="already in use"): + adapter.create_volume("sb-dup", GIB) diff --git a/tests/unit/storage_arrays/test_framework.py b/tests/unit/storage_arrays/test_framework.py new file mode 100644 index 0000000000..ef3b4e7360 --- /dev/null +++ b/tests/unit/storage_arrays/test_framework.py @@ -0,0 +1,266 @@ +"""Vendor-neutral pieces: registry, naming/sizing, config validation, +RestSession error mapping and retries, HTTPS mock server end to end.""" +from __future__ import annotations + +import threading +import uuid + +import pytest +import requests +from pydantic import SecretStr, ValidationError + +from simplyblock_core.credentials import Credential +from simplyblock_core.storage_arrays.base import ( + ArrayAuthError, + ArrayConfig, + ArrayConflictError, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + HostSpec, + Initiator, + Protocol, + colon_wwpn, + normalize_wwpn, +) +from simplyblock_core.storage_arrays.client import ( + ArrayClient, + credential_context, + managed_name, + round_size, +) +from simplyblock_core.storage_arrays.http import RestSession +from simplyblock_core.storage_arrays.registry import ADAPTERS, ALIASES, canonical_vendor +from simplyblock_core.storage_arrays.testing import ( + FakeArray, + FakeTransport, + Reply, + route, + serve, +) +from simplyblock_core.storage_arrays.testing.pure import FakePureFlashArray + +from .conftest import StaticProvider + + +def _config(**kw): + base = dict(id="a1", vendor="pure", endpoint="https://pure01.example.com", + credential_ref={"kind": "vault-kv", "mount": "m", "path": "p"}) + base.update(kw) + return ArrayConfig(**base) + + +class TestRegistry: + + def test_all_eleven_systems_registered(self): + assert len(ADAPTERS) == 11 + + def test_aliases_resolve(self): + for alias, target in ALIASES.items(): + assert canonical_vendor(alias) == target + assert canonical_vendor(" Pure ") == "pure-flasharray" + + def test_unknown_vendor(self): + with pytest.raises(ArrayUnsupportedError, match="supported"): + canonical_vendor("acme-array") + + +class TestNamingAndSizing: + + def test_managed_name_fits_31_chars(self): + u = str(uuid.uuid4()) + assert managed_name(u, 31) == "sb-" + uuid.UUID(u).hex[:24] + assert len(managed_name(u, 20)) == 20 + + def test_managed_name_requires_uuid(self): + with pytest.raises(ValueError): + managed_name("not-a-uuid", 63) + + @pytest.mark.parametrize("size, gran, minimum, expected", [ + (1, 512, 1 << 20, 1 << 20), + ((1 << 30) + 1, 512, 1 << 20, (1 << 30) + 512), + (10 << 30, 8 << 30, 8 << 30, 16 << 30), + (8 << 30, 8 << 30, 8 << 30, 8 << 30), + ]) + def test_round_size(self, size, gran, minimum, expected): + assert round_size(size, gran, minimum) == expected + + def test_credential_context_normalizes_endpoint(self): + assert credential_context(_config(endpoint="https://PURE01.example.com/")) == \ + "array/a1/https://pure01.example.com:443" + assert credential_context(_config(endpoint="https://u.example:8443/univmax")) == \ + "array/a1/https://u.example:8443" + + +class TestConfigAndModel: + + def test_https_only(self): + with pytest.raises(ValidationError): + _config(endpoint="http://pure01.example.com") + + def test_wwpn_forms(self): + assert normalize_wwpn("21:00:00:24:FF:4C:1A:2B") == "21000024ff4c1a2b" + assert normalize_wwpn("0x21000024ff4c1a2b") == "21000024ff4c1a2b" + assert colon_wwpn("21000024FF4C1A2B") == "21:00:00:24:ff:4c:1a:2b" + with pytest.raises(ValueError): + normalize_wwpn("2100") + + def test_host_cannot_mix_scsi_and_nvme(self): + with pytest.raises(ValidationError): + HostSpec(name="h", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.x"), + Initiator(protocol=Protocol.NVME_TCP, address="nqn.x"))) + HostSpec(name="h", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.x"), + Initiator(protocol=Protocol.FC, address="2100000000000001"))) + + def test_unknown_option_is_refused(self): + provider = StaticProvider(Credential({"api_token": SecretStr("pure-token")})) + client = ArrayClient(_config(options={"bogus": 1}), provider, # type: ignore[arg-type] + transport=FakeTransport(FakePureFlashArray())) + with pytest.raises(Exception, match="Unknown .* options: bogus"): + client.get_info() + + def test_wrong_credential_shape_is_refused(self): + provider = StaticProvider(Credential({"username": SecretStr("u"), + "password": SecretStr("p")})) + client = ArrayClient(_config(), provider, # type: ignore[arg-type] + transport=FakeTransport(FakePureFlashArray())) + with pytest.raises(ArrayAuthError, match="needs credential fields api_token"): + client.get_info() + + +class _Flaky(FakeArray): + def __init__(self): + super().__init__() + self.calls = 0 + + @route("GET", "/flaky") + def flaky(self, req): + self.calls += 1 + return Reply(503 if self.calls < 3 else 200, {"ok": self.calls}) + + @route("POST", "/flaky") + def flaky_post(self, req): + self.calls += 1 + return Reply(503, {}) + + @route("GET", "/status/(?P\\d+)") + def status(self, req, code): + return Reply(int(code), {"error": "x"}) + + +class TestRestSession: + + def _session(self, fake): + return RestSession("https://x.example", transport=FakeTransport(fake), + sleep=lambda s: None) + + def test_idempotent_requests_are_retried(self): + fake = _Flaky() + assert self._session(fake).get("/flaky").json() == {"ok": 3} + + def test_non_idempotent_requests_are_not_retried(self): + fake = _Flaky() + with pytest.raises(ArrayRequestError): + self._session(fake).post("/flaky") + assert fake.calls == 1 + + @pytest.mark.parametrize("code, exc", [ + (401, ArrayAuthError), (403, ArrayAuthError), (404, ArrayNotFoundError), + (409, ArrayConflictError), (400, ArrayRequestError), (500, ArrayRequestError), + ]) + def test_status_mapping(self, code, exc): + with pytest.raises(exc): + self._session(_Flaky()).get(f"/status/{code}") + + def test_ok_statuses_are_returned(self): + resp = self._session(_Flaky()).get("/status/400", ok=(400,)) + assert resp.status_code == 400 + + def test_ca_bundle_file_is_removed_on_close(self): + import os + session = RestSession.for_config(_config(ca_bundle="-----BEGIN CERTIFICATE-----\n")) + path = session.http.verify + assert isinstance(path, str) and os.path.exists(path) + session.close() + assert not os.path.exists(path) + + def test_verify_false_only_when_configured(self): + assert RestSession.for_config(_config()).http.verify is True + assert RestSession.for_config(_config(verify_tls=False)).http.verify is False + + +class TestHttpsMockServer: + + def test_end_to_end_over_tls_with_pinned_ca(self): + fake = FakePureFlashArray(api_token="t0k") + server = serve(fake, "127.0.0.1", 0) + port = server.server_address[1] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with open(server.certfile) as f: + ca = f.read() + provider = StaticProvider(Credential({"api_token": SecretStr("t0k")})) + config = _config(endpoint=f"https://127.0.0.1:{port}", ca_bundle=ca) + with ArrayClient(config, provider) as client: # type: ignore[arg-type] + vol = client.create_volume(str(uuid.uuid4()), 1 << 30) + assert client.get_volume(vol.id) == vol + assert client.delete_volume(vol.id) + + # Without the pinned CA the self-signed server is refused. + with ArrayClient(_config(endpoint=f"https://127.0.0.1:{port}"), + provider) as client: # type: ignore[arg-type] + with pytest.raises(ArrayRequestError, match="SSLError"): + client.get_info() + finally: + server.shutdown() + server.server_close() + + def test_requests_session_is_plain(self): + # FakeTransport is mounted per session, never globally. + assert not isinstance(requests.Session().get_adapter("https://x"), FakeTransport) + + +# Options set through Kubernetes arrive as strings. Each is read as the type of +# its default, so "false" is false rather than a truthy string. +def test_string_options_are_read_as_the_type_of_their_default(): + from unittest.mock import MagicMock + + from pydantic import SecretStr + + from simplyblock_core.credentials import Credential + from simplyblock_core.storage_arrays.adapters.dell_powermax import ( + DellPowerMaxAdapter, + ) + from simplyblock_core.storage_arrays.adapters.hitachi_vsp import HitachiVspAdapter + from simplyblock_core.storage_arrays.adapters.pure import PureFlashArrayAdapter + from simplyblock_core.storage_arrays.base import AdapterContext, ArrayError + from simplyblock_core.storage_arrays.testing import make_config + + def adapter(cls, vendor, creds, **options): + return cls(AdapterContext(make_config(vendor, "https://a.example", **options), + Credential({k: SecretStr(v) for k, v in creds.items()}), MagicMock())) + + pure = adapter(PureFlashArrayAdapter, "pure-flasharray", {"api_token": "t"}, eradicate="false") + assert pure.option("eradicate") is False + assert adapter(PureFlashArrayAdapter, "pure-flasharray", {"api_token": "t"}, + eradicate="True").option("eradicate") is True + with pytest.raises(ArrayError, match="eradicate"): + adapter(PureFlashArrayAdapter, "pure-flasharray", {"api_token": "t"}, eradicate="maybe") + + up = {"username": "u", "password": "p"} + vsp = adapter(HitachiVspAdapter, "hitachi-vsp", up, pool_id="1", ports="CL1-A", + host_mode_options="2, 13", job_timeout_s="60", use_lock="0") + assert vsp.option("host_mode_options") == ["2", "13"] + assert vsp.option("job_timeout_s") == 60 + assert vsp.option("use_lock") is False + + pmax = adapter(DellPowerMaxAdapter, "dell-powermax", up, symmetrix_id="000197900123", + port_group='{"iscsi": "pg_iscsi", "fc": "pg_fc"}', async_jobs="no") + assert pmax.option("port_group") == {"iscsi": "pg_iscsi", "fc": "pg_fc"} + assert pmax.option("async_jobs") is False + plain = adapter(DellPowerMaxAdapter, "dell-powermax", up, symmetrix_id="000197900123", + port_group="pg_all") + assert plain.option("port_group") == "pg_all" diff --git a/tests/unit/storage_arrays/test_hitachi_vsp.py b/tests/unit/storage_arrays/test_hitachi_vsp.py new file mode 100644 index 0000000000..7da110a84d --- /dev/null +++ b/tests/unit/storage_arrays/test_hitachi_vsp.py @@ -0,0 +1,425 @@ +"""Hitachi VSP specifics beyond the shared contract: path layouts, async +jobs, LDEV labels and paging, per-port host groups, LUN paths, locks.""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.adapters import hitachi_vsp as vsp +from simplyblock_core.storage_arrays.base import ( + ArrayConflictError, + ArrayError, + ArrayRequestError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeTransport +from simplyblock_core.storage_arrays.testing.hitachi_vsp import contract_setup + +from .conftest import StaticProvider + +GIB = 1 << 30 +FC_A = Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b") +FC_B = Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c") +IQN_A = Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a") + + +def _client(**options): + setup = contract_setup(**options) + sleeps: list[float] = [] + client = ArrayClient(setup.config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": sleeps.append}) + return client, setup.fake, sleeps + + +@pytest.fixture() +def env(): + client, fake, sleeps = _client() + yield client, fake, sleeps + client.close() + + +def _uuid() -> str: + return str(uuid.uuid4()) + + +def test_embedded_paths_session_header_and_logout(env): + client, fake, _ = env + client.get_info() + assert ("POST", "/ConfigurationManager/v1/objects/sessions") in fake.requests + assert ("GET", "/ConfigurationManager/v1/objects/storages/instance") in fake.requests + assert len(fake.sessions) == 1 + client.close() + assert fake.sessions == {} + assert fake.requests[-1][0] == "DELETE" + assert fake.requests[-1][1].startswith("/ConfigurationManager/v1/objects/sessions/") + + +def test_info_fields(env): + client, fake, _ = env + info = client.get_info() + assert info.model == "VSP E990" + assert info.serial == str(fake.serial) + assert info.version.startswith("93-") + + +def test_server_mode_uses_storage_device_prefix(): + setup_fake = contract_setup().fake + client, fake, _ = _client(storage_device_id=setup_fake.storage_device_id) + try: + prefix = f"/ConfigurationManager/v1/objects/storages/{fake.storage_device_id}" + info = client.get_info() + assert info.serial == str(fake.serial) + vol = client.create_volume(_uuid(), GIB) + host = client.ensure_host(HostSpec(name="sbtest-srv", initiators=(FC_A,))) + client.map_volume(vol.id, host) + assert client.delete_volume(vol.id) + paths = [p for _, p in fake.requests] + assert all(p.startswith(prefix) for p in paths) + assert any(p.startswith(f"{prefix}/jobs/") for p in paths) + assert not any(p.startswith("/ConfigurationManager/v1/objects/ldevs") for p in paths) + finally: + client.close() + + +def test_wrong_storage_device_id_is_not_found(): + client, _, _ = _client(storage_device_id="938000999999") + try: + with pytest.raises(ArrayError): + client.get_info() + finally: + client.close() + + +def test_create_waits_for_jobs_and_labels_ldev(env): + client, fake, sleeps = env + fake.job_polls = 3 + vol = client.create_volume(_uuid(), 10 * GIB + 1) + lid = int(vol.id) + assert fake.ldevs[lid]["label"] == vol.name + assert vol.size_bytes == 10 * GIB + (1 << 20) # 1 MiB granularity + assert fake.ldevs[lid]["blocks"] * 512 == vol.size_bytes + assert vol.wwn == "naa." + fake.naa(lid).lower() + assert sleeps, "job polling must go through the session's sleep" + methods = [m for m, p in fake.requests if "/jobs/" in p] + assert len(methods) >= 2 * 4 # 2 jobs x (3 running + 1 completed) + + +def test_job_timeout_is_reported_without_real_sleep(): + client, fake, sleeps = _client(job_timeout_s=10) + try: + client.get_info() + fake.job_polls = 10**6 + with pytest.raises(ArrayRequestError, match="not completed after 10s"): + client.create_volume(_uuid(), GIB) + assert 10 <= sum(sleeps) < 20 + finally: + client.close() + + +def test_failed_job_maps_storage_error(env): + client, fake, _ = env + vol = client.create_volume(_uuid(), GIB) + host = client.ensure_host(HostSpec(name="sbtest-a", initiators=(FC_A,))) + client.map_volume(vol.id, host) + # LDEV delete with an LU path left fails inside the job (KART30000-E + SSB) + adapter = client._adapter + assert adapter is not None + with pytest.raises(ArrayConflictError, match="KART30000-E.*LU paths"): + adapter.delete_volume(vol.id) + assert int(vol.id) in fake.ldevs + + +def test_deleted_ldev_reads_not_defined(env): + client, fake, _ = env + vol = client.create_volume(_uuid(), GIB) + client.delete_volume(vol.id) + assert int(vol.id) not in fake.ldevs + assert client.get_volume(vol.id) is None + assert client.get_volume("not-a-number") is None + + +def test_label_failure_deletes_the_new_ldev(env): + client, fake, _ = env + client.get_info() + fake.fail_next(500, {"errorSource": "x", "message": "label failed", + "messageId": "KART30000-E"}, path=r"/ldevs/\d+$") + with pytest.raises(ArrayRequestError, match="label failed"): + client.create_volume(_uuid(), GIB) + assert fake.ldevs == {} + + +def test_find_pages_through_ldevs(env, monkeypatch): + # count is a window of LDEV numbers: the head steps by count and the scan + # ends at the first window past the last LDEV number (KART40044-E). + client, fake, _ = env + monkeypatch.setattr(vsp, "PAGE_SIZE", 2) + fake.last_ldev = 9 + for i in range(5): + fake.add_ldev(f"other-{i}", GIB) + vol_uuid = _uuid() + vol = client.create_volume(vol_uuid, GIB) + assert int(vol.id) == 5 + fake.requests.clear() + assert client.find_volume(vol_uuid) == vol + heads = [p for m, p in fake.requests if m == "GET" and p.endswith("/ldevs")] + assert len(heads) == 6 # windows 0, 2, 4, 6, 8, then 10 is past the end + + +def test_find_scans_past_sparse_windows(env): + # An LDEV far above the first 16384 numbers is still found, although the + # first window returns (almost) nothing. + client, fake, _ = env + vol_uuid = _uuid() + name = client.volume_name(vol_uuid) + fake.add_ldev("other", GIB, ldev_id=3) + fake.add_ldev(name, GIB, ldev_id=40000) + vol = client.find_volume(vol_uuid) + assert vol is not None and vol.id == "40000" + gets = [p for m, p in fake.requests if m == "GET" and p.endswith("/ldevs")] + assert len(gets) == 4 # 0, 16384, 32768, 49152 up to LDEV 65535 + + +def test_find_first_window_error_is_not_swallowed(env): + client, fake, _ = env + client.get_info() + fake.fail_next(400, {"errorSource": "x", "message": "bad range", + "messageId": "KART40044-E"}, path=r"/ldevs$") + with pytest.raises(ArrayError, match="KART40044-E"): + client.find_volume(_uuid()) + + +def test_duplicate_labels_conflict(env): + client, fake, _ = env + vol_uuid = _uuid() + name = client.volume_name(vol_uuid) + fake.add_ldev(name, GIB) + fake.add_ldev(name, GIB) + with pytest.raises(ArrayConflictError, match="Several LDEVs"): + client.find_volume(vol_uuid) + + +def test_ldev_range_option(): + client, fake, _ = _client(ldev_range="100-101") + try: + fake.add_ldev("outside", GIB, ldev_id=5) + a = client.create_volume(_uuid(), GIB) + b = client.create_volume(_uuid(), GIB) + assert {a.id, b.id} == {"100", "101"} + with pytest.raises(ArrayError, match="No free LDEV"): + client.create_volume(_uuid(), GIB) + finally: + client.close() + + +@pytest.mark.parametrize("options, match", [ + ({"ports": []}, "ports"), + ({"ports": ["eth0"]}, "ports"), + ({"ldev_range": "9-3"}, "ldev_range"), + ({"pool_id": "x"}, "pool_id"), + ({"bogus": 1}, "Unknown"), +]) +def test_bad_options(options, match): + client, _, _ = _client(**options) + with pytest.raises(ArrayError, match=match): + client.get_info() + + +def test_host_is_a_group_per_port_of_its_protocol(env): + client, fake, _ = env + host = client.ensure_host(HostSpec(name="sbtest-fc", initiators=(FC_A,))) + groups = {k: g for k, g in fake.groups.items() if g["name"] == host} + assert sorted(p for p, _ in groups) == ["CL1-A", "CL2-A"] + assert all(g["wwns"] == ["21000024ff4c1a2b"] for g in groups.values()) + assert all(g["mode"] == "LINUX/IRIX" for g in groups.values()) + + iscsi = client.ensure_host(HostSpec(name="sbtest-is", initiators=(IQN_A,))) + assert sorted(p for (p, _), g in fake.groups.items() if g["name"] == iscsi) == \ + ["CL1-B", "CL2-B"] + + client.delete_host(host) + assert not any(g["name"] == host for g in fake.groups.values()) + assert ("CL1-A", 0) in fake.groups # default group untouched + + +def test_host_mode_options(): + client, fake, _ = _client(host_mode="VMWARE_EX", host_mode_options=[54, 63]) + try: + client.ensure_host(HostSpec(name="sbtest-vm", initiators=(FC_A,))) + g = next(g for g in fake.groups.values() if g["name"] == "sbtest-vm") + assert g["mode"] == "VMWARE_EX" and g["modeOptions"] == [54, 63] + finally: + client.close() + + +def test_no_port_for_protocol(): + client, _, _ = _client(ports=["CL1-B"]) + try: + with pytest.raises(ArrayError, match="no configured fc port"): + client.ensure_host(HostSpec(name="sbtest-fc", initiators=(FC_A,))) + finally: + client.close() + + +def test_iscsi_ownership_is_enforced_by_scan(env): + """The array lets one IQN into several targets of a port; the adapter must not.""" + client, fake, _ = env + client.ensure_host(HostSpec(name="sbtest-a", initiators=(IQN_A,))) + with pytest.raises(ArrayConflictError, match="sbtest-a"): + client.ensure_host(HostSpec(name="sbtest-b", initiators=(IQN_A,))) + assert not any(g["name"] == "sbtest-b" for g in fake.groups.values()) + + +def test_fc_wwn_in_unmanaged_group_conflicts(env): + client, fake, _ = env + client.get_info() + fake.groups[("CL2-A", 0)]["wwns"].append("21000024ff4c1a2c") + with pytest.raises(ArrayConflictError, match="CL2-A"): + client.ensure_host(HostSpec(name="sbtest-b", initiators=(FC_B,))) + + +def test_same_lun_on_every_port(env): + client, fake, _ = env + other = client.create_volume(_uuid(), GIB) + vol = client.create_volume(_uuid(), GIB) + host = client.ensure_host(HostSpec(name="sbtest-a", initiators=(FC_A,))) + # occupy LUN 0 on the second port only + num = next(n for (p, n), g in fake.groups.items() if p == "CL2-A" and g["name"] == host) + fake.groups[("CL2-A", num)]["luns"][0] = int(other.id) + m = client.map_volume(vol.id, host) + assert m.lun == 1 + luns = {p: lun for (p, n), g in fake.groups.items() if g["name"] == host + for lun, lid in g["luns"].items() if lid == int(vol.id)} + assert luns == {"CL1-A": 1, "CL2-A": 1} + assert client.list_mappings(vol.id) == [m] + + +def test_map_repairs_missing_port_path(env): + client, fake, _ = env + vol = client.create_volume(_uuid(), GIB) + host = client.ensure_host(HostSpec(name="sbtest-a", initiators=(FC_A,))) + m = client.map_volume(vol.id, host) + num = next(n for (p, n), g in fake.groups.items() if p == "CL2-A" and g["name"] == host) + fake.groups[("CL2-A", num)]["luns"].clear() + assert client.map_volume(vol.id, host) == m + assert fake.groups[("CL2-A", num)]["luns"] == {m.lun: int(vol.id)} + + +def test_unmap_removes_every_port_path(env): + client, fake, _ = env + vol = client.create_volume(_uuid(), GIB) + host = client.ensure_host(HostSpec(name="sbtest-a", initiators=(FC_A,))) + client.map_volume(vol.id, host) + client.unmap_volume(vol.id, host) + assert not any(g["luns"] for g in fake.groups.values()) + client.unmap_volume(vol.id, host) + + +def test_map_to_unknown_host_is_not_found(env): + client, _, _ = env + vol = client.create_volume(_uuid(), GIB) + with pytest.raises(ArrayError, match="no host group"): + client.map_volume(vol.id, "sbtest-missing") + + +def test_lock_required_without_use_lock_fails(): + client, fake, _ = _client() + try: + fake.require_lock = True + with pytest.raises(ArrayError, match="not locked"): + client.create_volume(_uuid(), GIB) + finally: + client.close() + + +def test_use_lock_wraps_changes(): + client, fake, _ = _client(use_lock=True) + try: + fake.require_lock = True + vol = client.create_volume(_uuid(), GIB) + host = client.ensure_host(HostSpec(name="sbtest-a", initiators=(FC_A,))) + client.map_volume(vol.id, host) + client.resize_volume(vol.id, 2 * GIB) + assert client.delete_volume(vol.id) + client.delete_host(host) + assert fake.lock_owner is None + locks = [p for _, p in fake.requests if p.endswith("/lock/invoke")] + unlocks = [p for _, p in fake.requests if p.endswith("/unlock/invoke")] + assert len(locks) == len(unlocks) >= 6 + finally: + client.close() + + +def test_lock_released_when_change_fails(): + client, fake, _ = _client(use_lock=True) + try: + client.get_info() + fake.require_lock = True + fake.pools.clear() + with pytest.raises(ArrayError): + client.create_volume(_uuid(), GIB) + assert fake.lock_owner is None + finally: + client.close() + + +def test_resize_expands_by_difference(env): + client, fake, _ = env + vol = client.create_volume(_uuid(), 2 * GIB) + grown = client.resize_volume(vol.id, 3 * GIB) + assert grown.size_bytes == 3 * GIB + assert fake.ldevs[int(vol.id)]["blocks"] == 3 * GIB // 512 + + +def test_classify_error_bodies(): + nf = vsp._classify("GET x", 400, {"message": "The LDEV is not defined.", + "messageId": "KART30000-E"}) + assert type(nf).__name__ == "ArrayNotFoundError" + cf = vsp._classify("POST x", None, {"message": "already registered", + "messageId": "KART30000-E", + "errorCode": {"SSB1": "B957", "SSB2": "4184"}}) + assert isinstance(cf, ArrayConflictError) and "SSB1=B957" in str(cf) + other = vsp._classify("POST x", 503, {"message": "busy"}) + assert isinstance(other, ArrayRequestError) and other.status == 503 + + +@pytest.mark.parametrize("body, kind", [ + # codes from Hitachi's Cinder driver (hbsd_rest_api.py / hbsd_rest.py) + ({"message": "x", "messageId": "KART30013-E"}, "ArrayNotFoundError"), + ({"message": "x", "messageId": "KART30000-E", "errorCode": {"errorCode": "EX_ENOOBJ"}}, + "ArrayNotFoundError"), + ({"message": "x", "messageId": "KART40050-E"}, "ArrayConflictError"), + ({"message": "x", "messageId": "KART30000-E", + "errorCode": {"SSB1": "2E22", "SSB2": "0001"}}, "ArrayConflictError"), + ({"message": "x", "messageId": "KART30000-E", + "errorCode": {"SSB1": "b958", "SSB2": "0947"}}, "ArrayConflictError"), + ({"message": "x", "messageId": "KART30000-E", + "errorCode": {"SSB1": "2E11", "SSB2": "2209"}}, "ArrayRequestError"), +]) +def test_classify_structured_codes(body, kind): + assert type(vsp._classify("POST x", 400, body)).__name__ == kind + + +def test_lock_uses_services_root(): + client, fake, _ = _client(use_lock=True) + try: + client.create_volume(_uuid(), GIB) + locks = [p for _, p in fake.requests if p.endswith("/lock/invoke")] + assert locks and all( + p == "/ConfigurationManager/v1/services/resource-group-service/actions/lock/invoke" + for p in locks) + finally: + client.close() + sdid = contract_setup().fake.storage_device_id + client, fake, _ = _client(use_lock=True, storage_device_id=sdid) + try: + client.create_volume(_uuid(), GIB) + locks = [p for _, p in fake.requests if p.endswith("/lock/invoke")] + assert locks and all(p.startswith(f"/ConfigurationManager/v1/{sdid}/services/") + for p in locks) + finally: + client.close() diff --git a/tests/unit/storage_arrays/test_hpe_nimble.py b/tests/unit/storage_arrays/test_hpe_nimble.py new file mode 100644 index 0000000000..04c395a954 --- /dev/null +++ b/tests/unit/storage_arrays/test_hpe_nimble.py @@ -0,0 +1,247 @@ +"""HPE Alletra 5000/6000 / Nimble adapter specifics beyond the shared contract.""" +from __future__ import annotations + +import dataclasses +import uuid + +import pytest + +from simplyblock_core.storage_arrays.base import ( + ArrayConflictError, + ArrayError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeTransport, make_config +from simplyblock_core.storage_arrays.testing.hpe_nimble import contract_setup + +from .conftest import StaticProvider + +GIB = 1 << 30 +MIB = 1 << 20 + + +def _client(**options): + setup = contract_setup() + if options: + setup = dataclasses.replace( + setup, config=make_config("hpe-nimble", "https://nimble.fake.example:5392", + **options)) + client = ArrayClient(setup.config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + return client, setup.fake + + +def _iscsi(name, *iqns): + return HostSpec(name=name, initiators=tuple( + Initiator(protocol=Protocol.ISCSI, address=i) for i in iqns)) + + +def _fc(name, *wwpns): + return HostSpec(name=name, initiators=tuple( + Initiator(protocol=Protocol.FC, address=w) for w in wwpns)) + + +def test_token_released_by_id_on_close(): + client, fake = _client() + client.get_info() + assert len(fake.tokens) == 1 + (tid,) = fake.tokens + client.close() + assert fake.tokens == {} + assert ("DELETE", f"/v1/tokens/{tid}") in fake.requests + + +def test_port_5392_is_kept(): + setup = contract_setup() + sent: list[str] = [] + + class Recording(FakeTransport): + def send(self, request, **kw): + sent.append(request.url) + return super().send(request, **kw) + + client = ArrayClient(setup.config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=Recording(setup.fake)) + client.get_info() + assert sent and all(u.startswith("https://nimble.fake.example:5392/v1/") for u in sent) + + +def test_get_info(): + client, _ = _client() + info = client.get_info() + assert info.model == "AF40" and info.serial == "AF-123456" + assert info.version.startswith("6.") + + +def test_sizes_in_mib_and_eui_identity(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB + 1) + assert vol.size_bytes == GIB + MIB + raw = fake.volumes[vol.id] + assert raw["size"] == 1025 + assert vol.nguid is None and vol.wwn == "eui." + raw["serial_number"] + assert vol.identifier == vol.wwn + + +def test_pool_and_performance_policy_resolved_by_name(): + client, fake = _client(pool="fast", performance_policy="Oracle OLTP") + vol = client.create_volume(str(uuid.uuid4()), GIB) + raw = fake.volumes[vol.id] + assert raw["pool_name"] == "fast" and raw["perfpolicy_name"] == "Oracle OLTP" + + +def test_unknown_pool_is_an_error(): + client, fake = _client(pool="nope") + with pytest.raises(ArrayError, match="pool 'nope' not found"): + client.create_volume(str(uuid.uuid4()), GIB) + assert fake.volumes == {} + + +def test_delete_offlines_first(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + adapter = client._adapter + assert adapter is not None + # a direct DELETE of an online volume is refused by the array + with pytest.raises(ArrayConflictError, match="SM_vol_online"): + adapter._request("DELETE", f"/v1/volumes/{vol.id}") + assert client.delete_volume(vol.id) is True + calls = [(m, p) for m, p in fake.requests if p == f"/v1/volumes/{vol.id}"] + assert ("PUT", f"/v1/volumes/{vol.id}") in calls + assert calls.index(("PUT", f"/v1/volumes/{vol.id}")) < len(calls) - 1 + + +def test_fc_group_uses_colon_wwpns(): + client, fake = _client() + hid = client.ensure_host(_fc("sbfc", "2100000E1E2F3A4B")) + assert fake.groups[hid]["access_protocol"] == "fc" + assert [i["wwpn"] for i in fake.initiators.values()] == ["21:00:00:0e:1e:2f:3a:4b"] + assert client.ensure_host(_fc("sbfc", "21:00:00:0e:1e:2f:3a:4b")) == hid + assert len(fake.initiators) == 1 + + +def test_mixed_protocol_host_refused(): + client, _ = _client() + spec = HostSpec(name="sbmix", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:m"), + Initiator(protocol=Protocol.FC, address="2100000e1e2f3a4c"))) + with pytest.raises(ArrayError, match="either iSCSI or FC"): + client.ensure_host(spec) + + +def test_existing_group_of_other_protocol_conflicts(): + client, _ = _client() + client.ensure_host(_iscsi("sbsame", "iqn.2026-01.io.sb:s")) + with pytest.raises(ArrayConflictError, match="access_protocol iscsi"): + client.ensure_host(_fc("sbsame", "2100000e1e2f3a4d")) + + +def test_initiator_owned_elsewhere_conflicts_before_creating_group(): + client, fake = _client() + client.ensure_host(_iscsi("sbown", "iqn.2026-01.io.sb:o")) + with pytest.raises(ArrayConflictError, match="sbown"): + client.ensure_host(_iscsi("sbthief", "iqn.2026-01.io.sb:o")) + assert {g["name"] for g in fake.groups.values()} == {"sbown"} + + +def test_iscsi_luns_are_zero_with_volume_scoped_targets(): + client, fake = _client() + hid = client.ensure_host(_iscsi("sbvst", "iqn.2026-01.io.sb:v")) + luns = [client.map_volume(client.create_volume(str(uuid.uuid4()), GIB).id, hid).lun + for _ in range(2)] + assert luns == [0, 0] + + +def test_fc_luns_are_distinct_per_group(): + client, fake = _client() + hid = client.ensure_host(_fc("sbfc2", "2100000e1e2f3a4e")) + luns = [client.map_volume(client.create_volume(str(uuid.uuid4()), GIB).id, hid).lun + for _ in range(3)] + assert luns == [0, 1, 2] + + +def test_acr_is_applied_to_both_and_map_is_idempotent(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + hid = client.ensure_host(_iscsi("sbacr", "iqn.2026-01.io.sb:a")) + m = client.map_volume(vol.id, hid) + assert client.map_volume(vol.id, hid) == m + (acr,) = fake.acrs.values() + assert acr["apply_to"] == "both" and acr["vol_id"] == vol.id + + +def test_malformed_volume_id_is_none(): + client, _ = _client() + assert client.get_volume("not-a-nimble-id") is None + + +def test_host_in_use_cannot_be_deleted(): + client, _ = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + hid = client.ensure_host(_iscsi("sbbusy", "iqn.2026-01.io.sb:b")) + client.map_volume(vol.id, hid) + with pytest.raises(ArrayConflictError, match="SM_igrp_has_acl"): + client.delete_host(hid) + + +def test_error_body_has_generic_then_specific_message(): + client, _ = _client() + client.ensure_host(_iscsi("sbdup", "iqn.2026-01.io.sb:d1")) + adapter = client._adapter + assert adapter is not None + with pytest.raises(ArrayConflictError, match="SM_http_conflict: .*; SM_eexist"): + adapter._data("POST", "/v1/initiator_groups", + json={"data": {"name": "sbdup", "access_protocol": "iscsi"}}) + + +def test_retried_delete_tolerates_already_offline_volume(): + # Offlining an offline volume answers "Object is already in requested + # state" (Cinder ignores it); a delete retried after the offline step + # must not fail on it. + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + fake.volumes[vol.id]["online"] = False + assert client.delete_volume(vol.id) is True + assert vol.id not in fake.volumes + + +def test_shrink_is_refused_without_force(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), 2 * GIB) + adapter = client._adapter + assert adapter is not None + with pytest.raises(ArrayError, match="SM_vol_size_decreased"): + adapter.resize_volume(vol.id, GIB) + assert fake.volumes[vol.id]["size"] == 2048 + + +def test_async_jobs_are_awaited_and_objects_reread(): + client, fake = _client() + fake.async_ops = {"POST volume", "PUT volume", "DELETE volume", "POST initiator_group", + "POST initiator", "POST acr", "DELETE acr", "DELETE initiator_group"} + vol = client.create_volume(str(uuid.uuid4()), GIB) + assert vol.size_bytes == GIB and vol.id in fake.volumes + assert client.resize_volume(vol.id, 2 * GIB).size_bytes == 2 * GIB + hid = client.ensure_host(_iscsi("sbasync", "iqn.2026-01.io.sb:as")) + assert len(fake.initiators) == 1 + m = client.map_volume(vol.id, hid) + assert m.lun == 0 and m.host_id == hid + client.unmap_volume(vol.id, hid) + assert fake.acrs == {} + assert client.delete_volume(vol.id) is True + client.delete_host(hid) + assert fake.volumes == {} and fake.groups == {} + polls = [p for m_, p in fake.requests if m_ == "GET" and p.startswith("/v1/jobs/")] + assert len(polls) >= 3 * len(fake.jobs) and all(v == 0 for v in fake.jobs.values()) + + +def test_async_job_timeout(): + client, fake = _client(job_timeout_s=4) + fake.async_ops = {"POST volume"} + fake.job_polls = 100 + with pytest.raises(ArrayError, match="did not finish within 4"): + client.create_volume(str(uuid.uuid4()), GIB) diff --git a/tests/unit/storage_arrays/test_hpe_wsapi.py b/tests/unit/storage_arrays/test_hpe_wsapi.py new file mode 100644 index 0000000000..366b647d24 --- /dev/null +++ b/tests/unit/storage_arrays/test_hpe_wsapi.py @@ -0,0 +1,243 @@ +"""HPE WSAPI adapter specifics beyond the shared contract.""" +from __future__ import annotations + +import dataclasses +import uuid + +import pytest + +from simplyblock_core.storage_arrays.base import ( + ArrayConflictError, + ArrayError, + ArrayNotFoundError, + ArrayRequestError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import ( + FakeError, + FakeTransport, + make_config, +) +from simplyblock_core.storage_arrays.testing.hpe_wsapi import contract_setup + +from .conftest import StaticProvider + +GIB = 1 << 30 +MIB = 1 << 20 + + +def _client(**options): + setup = contract_setup() + if options: + setup = dataclasses.replace( + setup, config=make_config("hpe-wsapi", "https://wsapi.fake.example", + **{"cpg": "SSD_r6", **options})) + client = ArrayClient(setup.config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + return client, setup.fake + + +def _iscsi(name, *iqns): + return HostSpec(name=name, initiators=tuple( + Initiator(protocol=Protocol.ISCSI, address=i) for i in iqns)) + + +def test_session_key_header_and_logout(): + client, fake = _client() + client.get_info() + assert len(fake.sessions) == 1 + client.close() + assert fake.sessions == set() + assert any(p.startswith("/api/v1/credentials/") for m, p in fake.requests if m == "DELETE") + + +def test_get_info_reads_system(): + client, _ = _client() + info = client.get_info() + assert info.model.startswith("HPE Alletra") and info.serial == "CZ12345678" + + +def test_cpg_option_required(): + from simplyblock_core.storage_arrays.adapters.hpe_wsapi import HpeWsapiAdapter + from simplyblock_core.storage_arrays.base import AdapterContext + from simplyblock_core.storage_arrays.http import RestSession + config = make_config("hpe-wsapi", "https://wsapi.fake.example") + ctx = AdapterContext(config, contract_setup().credential, RestSession(config.endpoint)) + with pytest.raises(ArrayError, match="cpg"): + HpeWsapiAdapter(ctx) + + +def test_invalid_provisioning_rejected(): + with pytest.raises(ArrayError, match="provisioning"): + _client(provisioning="full")[0].get_info() + + +def test_create_body_tpvv_and_reduce(): + client, fake = _client() + client.create_volume(str(uuid.uuid4()), GIB) + body = fake.created_bodies[-1] + assert body["tpvv"] is True and "reduce" not in body + assert body["cpg"] == "SSD_r6" and body["sizeMiB"] == 1024 + + client, fake = _client(provisioning="reduce") + client.create_volume(str(uuid.uuid4()), GIB) + assert fake.created_bodies[-1]["reduce"] is True and "tpvv" not in fake.created_bodies[-1] + + +def test_names_fit_31_chars_and_wwn_is_naa(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + assert len(vol.name) <= 31 and vol.id == vol.name + assert vol.wwn == "naa." + fake.volumes[vol.name]["wwn"].lower() + assert vol.wwn.startswith("naa.60002ac") + + +def test_size_rounds_to_256_mib(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB + 1) + assert vol.size_bytes == GIB + 256 * MIB + assert fake.volumes[vol.name]["sizeMiB"] == 1280 + + +def test_grow_sends_increment(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), 2 * GIB) + seen = [] + orig = fake.put_volume + + def spy(req, name): + seen.append(req.json()) + return orig(req, name) + + fake._routes = [(m, p, spy if h == orig else h) for m, p, h in fake._routes] + grown = client.resize_volume(vol.id, 5 * GIB) + assert seen == [{"action": 3, "sizeMiB": 3 * 1024}] + assert grown.size_bytes == 5 * GIB + + +def test_delete_with_vlun_is_conflict_at_adapter_level(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + host = client.ensure_host(_iscsi("sbh1", "iqn.2026-01.io.sb:x")) + client.map_volume(vol.id, host) + adapter = client._adapter + assert adapter is not None + with pytest.raises(ArrayConflictError, match="code 34"): + adapter.delete_volume(vol.id) + # the client unmaps first + assert client.delete_volume(vol.id) is True + + +def test_vlun_location_and_dedup_of_active_paths(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + h = client.ensure_host(_iscsi("sbh2", "iqn.2026-01.io.sb:y")) + fake.vluns.append({"volumeName": "other", "hostname": h, "lun": 0}) # LUN 0 taken + fake.add_volume("other", GIB) + m = client.map_volume(vol.id, h) + assert m.lun == 1 + members = [x for x in fake._vlun_members(vol.id)] + assert len(members) == 1 + fake.paths_per_host + assert client.list_mappings(vol.id) == [m] + + +def test_missing_host_on_map_is_not_found(): + client, _ = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + with pytest.raises(ArrayNotFoundError, match="code 17"): + client.map_volume(vol.id, "nohost") + + +def test_mixed_iscsi_fc_host_uses_two_requests(): + client, fake = _client() + spec = HostSpec(name="sbmixed", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:m"), + Initiator(protocol=Protocol.FC, address="10:00:00:00:c9:aa:bb:cc"))) + assert client.ensure_host(spec) == "sbmixed" + host = fake.hosts["sbmixed"] + assert [p["name"] for p in host["iSCSIPaths"]] == ["iqn.2026-01.io.sb:m"] + assert [p["wwn"] for p in host["FCPaths"]] == ["10000000C9AABBCC"] + assert client.ensure_host(spec) == "sbmixed" + + +def test_persona_option_used(): + client, fake = _client(persona=11) + client.ensure_host(_iscsi("sbvm", "iqn.2026-01.io.sb:v")) + assert fake.hosts["sbvm"]["persona"] == 11 + + +def test_path_owned_by_other_host_is_conflict(): + client, _ = _client() + client.ensure_host(_iscsi("sba", "iqn.2026-01.io.sb:dup")) + with pytest.raises(ArrayConflictError, match="code 73"): + client.ensure_host(_iscsi("sbb", "iqn.2026-01.io.sb:dup")) + + +def test_nvme_refused(): + client, fake = _client() + spec = HostSpec(name="sbn", initiators=( + Initiator(protocol=Protocol.NVME_TCP, address="nqn.2014-08.org.nvmexpress:uuid:x"),)) + with pytest.raises(ArrayError, match="does not support"): + client.ensure_host(spec) + assert not fake.requests + + +def test_host_name_limit_enforced_by_client(): + client, fake = _client() + with pytest.raises(ArrayError, match="exceeds 31"): + client.ensure_host(_iscsi("h" * 32, "iqn.2026-01.io.sb:long")) + assert not fake.requests + + +def _logins(fake): + return sum(1 for m, p in fake.requests if m == "POST" and p == "/api/v1/credentials") + + +def test_non_auth_403_fails_without_relogin(): + # 403 code 153 (VV_NEW_SIZE_EXCEED_CPG_LIMIT) is a refusal, not an + # expired session: no re-login, no credential invalidation. + client, fake = _client() + fake.cpg_limit_mib = 2048 + vol = client.create_volume(str(uuid.uuid4()), GIB) + with pytest.raises(ArrayRequestError, match="code 153") as exc: + client.resize_volume(vol.id, 4 * GIB) + assert exc.value.status == 403 + assert _logins(fake) == 1 + + +def test_expired_session_still_relogs_in(): + client, fake = _client() + client.get_info() + fake.expire_sessions() + client.get_info() + assert _logins(fake) == 2 + + +def test_create_below_256_mib_is_refused_by_array(): + client, _ = _client() + adapter = client._build() # logs in + with pytest.raises(ArrayRequestError, match="code 43"): + adapter._call("POST", "/volumes", json={"name": "tiny", "cpg": "SSD_r6", + "sizeMiB": 255}) + + +def test_code_29_is_not_a_conflict(): + # Cinder sees code 29 as HTTP 400 on a volume delete (online copy), not + # as an existing VLUN. + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + + def refuse(req, name): + raise FakeError(400, {"code": 29, "desc": "volume has an online copy"}) + + fake._routes = [(m, p, refuse if h == fake.delete_volume else h) + for m, p, h in fake._routes] + adapter = client._adapter + assert adapter is not None + with pytest.raises(ArrayRequestError, match="code 29") as exc: + adapter.delete_volume(vol.id) + assert not isinstance(exc.value, ArrayConflictError) diff --git a/tests/unit/storage_arrays/test_huawei_dorado.py b/tests/unit/storage_arrays/test_huawei_dorado.py new file mode 100644 index 0000000000..51013a18c3 --- /dev/null +++ b/tests/unit/storage_arrays/test_huawei_dorado.py @@ -0,0 +1,195 @@ +"""Huawei OceanStor Dorado specifics beyond the shared contract suite.""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.adapters.huawei_dorado import HuaweiDoradoAdapter +from simplyblock_core.storage_arrays.base import ( + ArrayAuthError, + ArrayConflictError, + ArrayError, + ArrayRequestError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeTransport +from simplyblock_core.storage_arrays.testing.huawei_dorado import contract_setup + +GIB = 1 << 30 +IQN_A = Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:a") +IQN_B = Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:b") + + +class _Provider: + def __init__(self, credential): + self.credential = credential + self.invalidated = 0 + + def resolve(self, ref, context): + return self.credential + + def invalidate(self, ref=None): + self.invalidated += 1 + + +def _client(**options): + setup = contract_setup() + if options: + setup.config = setup.config.model_copy( + update={"options": {**setup.config.options, **options}}) + client = ArrayClient(setup.config, _Provider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + return client, setup.fake + + +def _envelope(code, description="x"): + return {"data": {}, "error": {"code": code, "description": description}} + + +def test_login_and_device_scoped_paths(): + client, fake = _client() + info = client.get_info() + assert info.serial == fake.device_id and info.version == "6.1.5" + assert info.model == "OceanStor Dorado 5000 V6" + paths = [p for _, p in fake.requests] + assert paths[0] == "/deviceManager/rest/xx/sessions" + assert paths[1] == f"/deviceManager/rest/{fake.device_id}/system/" + client.close() + assert fake.requests[-1] == ("DELETE", f"/deviceManager/rest/{fake.device_id}/sessions") + assert not fake.tokens + + +def test_capacity_is_in_sectors_and_rounded_to_mib(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), 10 * GIB + 1) + assert vol.size_bytes == 10 * GIB + (1 << 20) + assert fake.luns[vol.id]["CAPACITY"] == str((10 * GIB + (1 << 20)) // 512) + assert fake.luns[vol.id]["ALLOCTYPE"] == "1" + assert vol.wwn == "naa." + fake.luns[vol.id]["WWN"].lower() + assert len(vol.name) <= 31 + + +def test_unknown_pool(): + client, _ = _client(storage_pool="nope") + with pytest.raises(ArrayError, match="storage pool 'nope' not found"): + client.create_volume(str(uuid.uuid4()), GIB) + + +def test_http_200_error_envelopes_are_mapped(): + client, fake = _client() + client.get_info() + fake.fail_next(200, _envelope(1077948993, "The name already exists."), path="/lun$") + with pytest.raises(ArrayConflictError, match="1077948993"): + client.find_volume(str(uuid.uuid4())) + fake.fail_next(200, _envelope(1077949006, "The system is busy."), path="/lun$") + with pytest.raises(ArrayRequestError, match="1077949006"): + client.find_volume(str(uuid.uuid4())) + fake.fail_next(200, _envelope(1077936859, "The LUN does not exist."), path="/lun/") + assert client.get_volume("4242") is None + + +def test_minus_401_envelope_relogs_in_once(): + client, fake = _client() + client.get_info() + fake.expire_sessions() + client.get_info() + logins = [p for _, p in fake.requests if p.endswith("/xx/sessions")] + assert len(logins) == 2 + + +def test_wrong_password_is_auth_error(): + client, fake = _client() + fake.set_credential(username="admin", password="other") + with pytest.raises(ArrayAuthError, match="1077987870"): + client.get_info() + + +def test_mapping_triple_and_teardown(): + client, fake = _client() + v1 = client.create_volume(str(uuid.uuid4()), GIB) + v2 = client.create_volume(str(uuid.uuid4()), GIB) + host = client.ensure_host(HostSpec(name="sbtest-a", initiators=(IQN_A,))) + m1 = client.map_volume(v1.id, host) + m2 = client.map_volume(v2.id, host) + assert (m1.lun, m2.lun) == (1, 2) # HostLUNID parsed from ASSOCIATEMETADATA + assert {g["NAME"] for g in fake.lungroups.values()} == {f"sb_lg_{host}"} + assert {g["NAME"] for g in fake.hostgroups.values()} == {f"sb_hg_{host}"} + assert {v["NAME"] for v in fake.views.values()} == {f"sb_mv_{host}"} + assert client.map_volume(v1.id, host) == m1 + + with pytest.raises(ArrayConflictError, match="still has LUNs"): + client.delete_host(host) + client.delete_volume(v1.id) + client.delete_volume(v2.id) + client.delete_host(host) + assert not (fake.hosts or fake.lungroups or fake.hostgroups or fake.views) + assert fake.initiators["iscsi_initiator"]["iqn.2026-01.io.sb:a"]["ISFREE"] == "true" + # the freed initiator can be claimed by a new host + client.ensure_host(HostSpec(name="sbtest-b", initiators=(IQN_A,))) + + +def test_initiator_bound_to_foreign_host_creates_nothing(): + client, fake = _client() + owner = client.ensure_host(HostSpec(name="customer-host", initiators=(IQN_A,))) + with pytest.raises(ArrayConflictError, match="customer-host"): + client.ensure_host(HostSpec(name="sbtest-x", initiators=(IQN_B, IQN_A))) + assert [h["NAME"] for h in fake.hosts.values()] == ["customer-host"] + assert fake.initiators["iscsi_initiator"]["iqn.2026-01.io.sb:a"]["PARENTID"] == owner + + +def test_fc_initiator_ids_are_plain_lowercase_hex(): + client, fake = _client() + host = client.ensure_host(HostSpec(name="sbtest-fc", initiators=( + Initiator(protocol=Protocol.FC, address="21:00:00:24:FF:4C:1A:2B"),))) + ini = fake.initiators["fc_initiator"]["21000024ff4c1a2b"] + assert ini["PARENTID"] == host and ini["ISFREE"] == "false" + + +def test_host_in_foreign_hostgroup_conflicts_on_map(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + host = client.ensure_host(HostSpec(name="sbtest-a", initiators=(IQN_A,))) + gid = str(fake.next_id()) + fake.hostgroups[gid] = {"ID": gid, "NAME": "customer_hg", "TYPE": "14", + "DESCRIPTION": "", "hosts": {host}} + with pytest.raises(ArrayConflictError, match="customer_hg"): + client.map_volume(vol.id, host) + + +def test_lun_in_customer_lungroup_is_not_listed_and_blocks_delete(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + gid = str(fake.next_id()) + fake.lungroups[gid] = {"ID": gid, "NAME": "customer_lg", "TYPE": "256", + "DESCRIPTION": "", "luns": {vol.id}} + assert client.list_mappings(vol.id) == [] + with pytest.raises(ArrayRequestError): + client.delete_volume(vol.id) + assert client.get_volume(vol.id) is not None + + +def test_name_query_keeps_separator_literal(): + # Huawei's CSI driver and Cinder send filter=NAME::&range=[0-100] + # with a literal "::"; only the name is escaped. + q = HuaweiDoradoAdapter._name_query("/lun", "sb-a b/c") + assert q == "/lun?filter=NAME::sb-a%20b%2Fc&range=[0-100]" + client, _ = _client() + vol_uuid = str(uuid.uuid4()) + vol = client.create_volume(vol_uuid, GIB) + assert client.find_volume(vol_uuid) == vol + + +def test_delete_host_never_unbinds_foreign_initiators(): + client, fake = _client() + fake.ignore_parent_filter = True + keep = client.ensure_host(HostSpec(name="sbtest-keep", initiators=(IQN_A,))) + gone = client.ensure_host(HostSpec(name="sbtest-gone", initiators=(IQN_B,))) + client.delete_host(gone) + ini = fake.initiators["iscsi_initiator"][IQN_A.address] + assert ini["ISFREE"] == "false" and ini["PARENTID"] == keep + assert fake.initiators["iscsi_initiator"][IQN_B.address]["ISFREE"] == "true" diff --git a/tests/unit/storage_arrays/test_ibm_flashsystem.py b/tests/unit/storage_arrays/test_ibm_flashsystem.py new file mode 100644 index 0000000000..0c90f09e70 --- /dev/null +++ b/tests/unit/storage_arrays/test_ibm_flashsystem.py @@ -0,0 +1,216 @@ +"""IBM Storage Virtualize specifics beyond the shared contract suite.""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.base import ( + ArrayConflictError, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeTransport +from simplyblock_core.storage_arrays.testing.ibm_flashsystem import contract_setup + +GIB = 1 << 30 + + +class _Provider: + def __init__(self, credential): + self.credential = credential + self.invalidated = 0 + + def resolve(self, ref, context): + return self.credential + + def invalidate(self, ref=None): + self.invalidated += 1 + + +def _client(**options): + setup = contract_setup() + if options: + setup.config = setup.config.model_copy( + update={"options": {**setup.config.options, **options}}) + client = ArrayClient(setup.config, _Provider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + return client, setup.fake + + +def _cmds(fake, name): + return [opts for cmd, _, opts in fake.commands if cmd == name] + + +def test_get_info(): + client, _ = _client() + info = client.get_info() + assert info.model == "IBM FlashSystem 7300" + assert info.version == "8.6.0.4" + assert info.serial == "000002042140A0B2" + + +def test_mkvolume_parameters_thin_default(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), 10 * GIB + 1) + (opts,) = _cmds(fake, "mkvolume") + assert opts == {"name": vol.name, "pool": "Pool0", "size": 10 * GIB + 512, + "unit": "b", "thin": True} + assert vol.size_bytes == 10 * GIB + 512 + assert vol.wwn == "naa." + fake.vdisks[vol.id]["vdisk_UID"].lower() + # capacity must be read in bytes, never in the human-readable form + assert all(o.get("bytes") is True for o in _cmds(fake, "lsvdisk")) + + +def test_thick_volume_and_io_group(): + client, fake = _client(thin=False, io_group="io_grp1") + vol = client.create_volume(str(uuid.uuid4()), GIB) + (opts,) = _cmds(fake, "mkvolume") + assert "thin" not in opts and opts["iogrp"] == "io_grp1" + assert fake.vdisks[vol.id]["IO_group_name"] == "io_grp1" + + +def test_rest_v1_root(): + client, fake = _client(rest_root="/rest/v1") + client.create_volume(str(uuid.uuid4()), GIB) + assert all(path.startswith("/rest/v1/") for _, path in fake.requests) + + +def test_resize_sends_delta(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), 8 * GIB) + client.resize_volume(vol.id, 20 * GIB) + assert _cmds(fake, "expandvdisksize") == [{"size": 12 * GIB, "unit": "b"}] + + +def test_unknown_pool_is_not_found(): + client, _ = _client(pool="NoSuchPool") + with pytest.raises(ArrayNotFoundError, match="CMMVC5754E"): + client.create_volume(str(uuid.uuid4()), GIB) + + +def test_cmmvc_codes_are_mapped(): + client, fake = _client() + fake.fail_next(500, b"CMMVC6035E The action failed as the object already exists.", + path="mkvolume") + with pytest.raises(ArrayConflictError, match="CMMVC6035E"): + client.create_volume(str(uuid.uuid4()), GIB) + fake.fail_next(500, b"CMMVC5786E The action failed because the cluster is not in a " + b"stable state.", path="mkvolume") + with pytest.raises(ArrayRequestError) as e: + client.create_volume(str(uuid.uuid4()), GIB) + assert e.value.status == 500 and "CMMVC5786E" in str(e.value) + + +def test_expired_token_is_403_and_relogs_in(): + client, fake = _client() + client.get_info() + fake.expire_sessions() + client.get_info() + assert [p for _, p in fake.requests].count("/rest/auth") == 2 + + +def test_volume_id_reuse_is_caught_by_identifier(): + client, _ = _client() + old = client.create_volume(str(uuid.uuid4()), GIB) + assert client.delete_volume(old.id) + new = client.create_volume(str(uuid.uuid4()), GIB) + assert new.id == old.id # the array hands out the lowest free id again + with pytest.raises(ArrayConflictError, match="does not match"): + client.delete_volume(old.id, expected_identifier=old.identifier) + assert client.get_volume(new.id) == new + + +def test_multi_host_mapping_uses_force_and_single_row_quirk(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + a = client.ensure_host(HostSpec(name="sbtest-a", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:a"),))) + b = client.ensure_host(HostSpec(name="sbtest-b", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:b"),))) + m = client.map_volume(vol.id, a) + assert m.lun == 0 + # one mapping: lsvdiskhostmap answers with a bare dict + assert client.list_mappings(vol.id) == [m] + client.map_volume(vol.id, b) + assert all(o.get("force") is True for o in _cmds(fake, "mkvdiskhostmap")) + assert {x.host_id for x in client.list_mappings(vol.id)} == {a, b} + + +def test_fc_host_wwpns_are_colon_separated_and_forced(): + client, fake = _client() + client.ensure_host(HostSpec(name="sbtest-fc", initiators=( + Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"), + Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c")))) + (opts,) = _cmds(fake, "mkhost") + assert opts == {"name": "sbtest-fc", "fcwwpn": "21000024FF4C1A2B:21000024FF4C1A2C", + "force": True} + # re-run with the same ports: nothing is re-added + client.ensure_host(HostSpec(name="sbtest-fc", initiators=( + Initiator(protocol=Protocol.FC, address="21000024ff4c1a2b"),))) + assert _cmds(fake, "addhostport") == [] + + +def test_fc_port_of_other_host_conflicts(): + client, fake = _client() + wwpn = Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b") + client.ensure_host(HostSpec(name="sbtest-one", initiators=(wwpn,))) + with pytest.raises(ArrayConflictError, match="registered to another host"): + client.ensure_host(HostSpec(name="sbtest-two", initiators=(wwpn,))) + assert "sbtest-two" not in fake.hosts + + +def test_nvme_host_protocol(): + client, fake = _client() + client.ensure_host(HostSpec(name="sbtest-nvme", initiators=( + Initiator(protocol=Protocol.NVME_TCP, address="nqn.2014-08.org.nvmexpress:uuid:x"),))) + assert fake.hosts["sbtest-nvme"]["protocol"] == "tcpnvme" + with pytest.raises(ArrayUnsupportedError, match="one transport"): + client.ensure_host(HostSpec(name="sbtest-mixed", initiators=( + Initiator(protocol=Protocol.NVME_TCP, address="nqn.2014-08.org.nvmexpress:uuid:y"), + Initiator(protocol=Protocol.NVME_RDMA, address="nqn.2014-08.org.nvmexpress:uuid:z")))) + + +def test_delete_host_with_mappings_conflicts(): + client, _ = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + host = client.ensure_host(HostSpec(name="sbtest-busy", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:busy"),))) + client.map_volume(vol.id, host) + with pytest.raises(ArrayConflictError, match="CMMVC5871E"): + client.delete_host(host) + + +def test_mixed_fc_iscsi_host_uses_one_port_kind_per_mkhost(): + # mkhost takes exactly one of -fcwwpn / -iscsiname / -nqn (Ansible + # ibm_svc_host, Cinder create_host); the rest goes through addhostport. + client, fake = _client() + client.ensure_host(HostSpec(name="sbtest-mix", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:mix"), + Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2d")))) + (mk,) = _cmds(fake, "mkhost") + assert mk == {"name": "sbtest-mix", "fcwwpn": "21000024FF4C1A2D", "force": True} + assert _cmds(fake, "addhostport") == [{"iscsiname": "iqn.2026-01.io.sb:mix", + "force": True}] + host = fake.hosts["sbtest-mix"] + assert host["fcwwpn"] == ["21000024FF4C1A2D"] + assert host["iscsiname"] == ["iqn.2026-01.io.sb:mix"] + + +@pytest.mark.parametrize("initiator, code", [ + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:shared"), "CMMVC6578E"), + (Initiator(protocol=Protocol.NVME_TCP, address="nqn.2014-08.org.nvmexpress:uuid:s"), + "CMMVC9328E"), +]) +def test_iscsi_and_nvme_port_of_other_host_conflicts(initiator, code): + client, fake = _client() + client.ensure_host(HostSpec(name="sbtest-one", initiators=(initiator,))) + with pytest.raises(ArrayConflictError, match=f"registered to another host.*{code}"): + client.ensure_host(HostSpec(name="sbtest-two", initiators=(initiator,))) + assert "sbtest-two" not in fake.hosts diff --git a/tests/unit/storage_arrays/test_infinidat.py b/tests/unit/storage_arrays/test_infinidat.py new file mode 100644 index 0000000000..742ac5d59b --- /dev/null +++ b/tests/unit/storage_arrays/test_infinidat.py @@ -0,0 +1,181 @@ +"""Infinidat InfiniBox specifics beyond the shared contract.""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.adapters.infinidat import infinibox_wwn +from simplyblock_core.storage_arrays.base import ( + ArrayConflictError, + ArrayError, + ArrayNotFoundError, + ArrayRequestError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeTransport, make_config +from simplyblock_core.storage_arrays.testing.infinidat import contract_setup + +from .conftest import StaticProvider + +GIB = 1 << 30 + + +@pytest.fixture() +def setup(): + return contract_setup() + + +def _client(setup, **options): + config = setup.config + if options: + config = make_config("infinidat-infinibox", config.endpoint, + **{**config.options, **options}) + return ArrayClient(config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + + +def _iqn(n: str) -> Initiator: + return Initiator(protocol=Protocol.ISCSI, address=f"iqn.2026-01.io.sb:{n}") + + +def test_wwn_from_serial(): + assert infinibox_wwn("742B0F000004E2B000000000000F3A3") == "naa.6742b0f000004e2b000000000000f3a3" + assert infinibox_wwn("6742b0f000004e2b000000000000f3a3") == "naa.6742b0f000004e2b000000000000f3a3" + + +def test_volume_identity_and_pool(setup): + with _client(setup) as c: + vol = c.create_volume(str(uuid.uuid4()), 10 * GIB + 1) + raw = setup.fake.volumes[int(vol.id)] + assert vol.wwn == "naa.6" + raw["serial"] + assert len(vol.wwn) == len("naa.") + 32 + assert raw["pool_id"] == next(iter(setup.fake.pools)) + assert raw["provtype"] == "THIN" + assert vol.size_bytes == 10 * GIB + (1 << 20) + + +def test_thick_option_and_bad_option(setup): + with _client(setup, provtype="thick") as c: + vol = c.create_volume(str(uuid.uuid4()), GIB) + assert setup.fake.volumes[int(vol.id)]["provtype"] == "THICK" + with _client(setup, provtype="sparse") as c, pytest.raises(ArrayError, match="provtype"): + c.get_info() + + +def test_missing_pool(setup): + with _client(setup, pool="nope") as c, pytest.raises(ArrayNotFoundError, match="nope"): + c.create_volume(str(uuid.uuid4()), GIB) + + +def test_destructive_requests_send_approval(setup): + with _client(setup) as c: + vol = c.create_volume(str(uuid.uuid4()), GIB) + host = c.ensure_host(HostSpec(name="h1", initiators=(_iqn("a"),))) + c.map_volume(vol.id, host) + assert c.delete_volume(vol.id) # unmap + delete, both need approved=true + c.delete_host(host) + assert not setup.fake.volumes and not setup.fake.hosts + + +def test_every_mutation_is_preapproved(setup): + # infinisdk sends approved=true on every non-GET request; so does the adapter. + setup.fake.approval_for_all = True + fc = Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b") + with _client(setup) as c: + vol = c.create_volume(str(uuid.uuid4()), GIB) + vol = c.resize_volume(vol.id, 2 * GIB) + host = c.ensure_host(HostSpec(name="h1", initiators=(_iqn("a"), fc))) + c.map_volume(vol.id, host) + c.unmap_volume(vol.id, host) + assert c.delete_volume(vol.id) + c.delete_host(host) + assert not setup.fake.volumes and not setup.fake.hosts + + +@pytest.mark.parametrize("code", ["APPROVAL_REQUIRED", "APPROVAL_REQUIRED_VOLUME_HAS_CHILDREN"]) +def test_specific_approval_codes_are_not_auth_errors(setup, code): + with _client(setup) as c: + vol = c.create_volume(str(uuid.uuid4()), GIB) + setup.fake.fail_next(403, {"result": None, "error": { + "code": code, "message": "needs approval", "reasons": ["has children"]}, + "metadata": {"ready": True}}, path=r"/volumes/\d+$") + adapter = c._adapter + assert adapter is not None + with pytest.raises(ArrayRequestError, match=code): + adapter.delete_volume(vol.id) + + +def test_get_info_reads_model_field(setup): + with _client(setup) as c: + info = c.get_info() + assert info.model == "F4304" and info.serial == "1250" and info.version == "7.3.10.0" + + +def test_approval_required_is_not_an_auth_error(setup): + with _client(setup) as c: + vol = c.create_volume(str(uuid.uuid4()), GIB) + setup.fake.fail_next(403, {"result": None, "error": { + "code": "APPROVAL_REQUIRED", "message": "needs approval"}, + "metadata": {"ready": True}}, path=r"/volumes/\d+$") + adapter = c._adapter + assert adapter is not None + with pytest.raises(ArrayRequestError, match="APPROVAL_REQUIRED"): + adapter.delete_volume(vol.id) + + +def test_conflicting_port_rolls_back_new_host(setup): + with _client(setup) as c: + c.ensure_host(HostSpec(name="h1", initiators=(_iqn("a"),))) + with pytest.raises(ArrayConflictError): + c.ensure_host(HostSpec(name="h2", initiators=(_iqn("b"), _iqn("a")))) + assert [h["name"] for h in setup.fake.hosts.values()] == ["h1"] + + +def test_fc_and_iscsi_ports_normalised(setup): + fc = Initiator(protocol=Protocol.FC, address="21:00:00:24:FF:4C:1A:2B") + with _client(setup) as c: + host = c.ensure_host(HostSpec(name="h1", initiators=(fc, _iqn("A")))) + ports = setup.fake.hosts[int(host)]["ports"] + assert {(p["type"], p["address"]) for p in ports} == { + ("FC", "21000024ff4c1a2b"), ("ISCSI", "iqn.2026-01.io.sb:a")} + # colon-formatted ports returned by the array still count as present + ports[0]["address"] = "21:00:00:24:ff:4c:1a:2b" + assert c.ensure_host(HostSpec(name="h1", initiators=(fc,))) == host + assert len(setup.fake.hosts[int(host)]["ports"]) == 2 + + +def test_luns_start_at_one_per_host(setup): + with _client(setup) as c: + host = c.ensure_host(HostSpec(name="h1", initiators=(_iqn("a"),))) + luns = [c.map_volume(c.create_volume(str(uuid.uuid4()), GIB).id, host).lun + for _ in range(3)] + assert luns == [1, 2, 3] + + +def test_pagination_walks_all_pages(setup, monkeypatch): + import simplyblock_core.storage_arrays.adapters.infinidat as mod + monkeypatch.setattr(mod, "_PAGE_SIZE", 2) + for i in range(5): + setup.fake.add_volume(f"sb-same-{i}", GIB) + with _client(setup) as c: + c.get_info() + adapter = c._adapter + assert adapter is not None + assert len(adapter._list("/volumes")) == 5 + + +def test_non_numeric_ids_are_absent(setup): + with _client(setup) as c: + assert c.get_volume("sb-abc") is None + c.delete_host("not-a-number") + + +def test_bad_credential_is_auth_error(setup): + setup.fake.set_credential(password="other") + with _client(setup) as c, pytest.raises(ArrayError): + c.get_info() diff --git a/tests/unit/storage_arrays/test_lio.py b/tests/unit/storage_arrays/test_lio.py new file mode 100644 index 0000000000..9d92107c02 --- /dev/null +++ b/tests/unit/storage_arrays/test_lio.py @@ -0,0 +1,105 @@ +"""The LIO data path of the mock arrays (storage_arrays.testing.lio): a fake's +volumes, hosts, and mappings mirrored into an iSCSI target, here into the +in-memory backend. Every vendor fake with iSCSI runs the same scenario through +its own adapter, which is what the control plane would send it.""" +from __future__ import annotations + +import pytest + +from simplyblock_core.array.san import host_wwn +from simplyblock_core.storage_arrays.base import HostSpec, Initiator, Protocol +from simplyblock_core.storage_arrays.testing import FakeTransport +from simplyblock_core.storage_arrays.testing.lio import LioMirror, MemoryLio + +from .conftest import VENDORS, Harness, fake_module + +GIB = 1 << 30 +ISCSI_VENDORS = [v for v in VENDORS if v != "dell-powerflex"] +UUID = "6f1d3a52-8c4b-4e0a-9d27-3b8e4c1f7a90" + + +class Bed: + def __init__(self, vendor: str): + self.setup = fake_module(vendor).contract_setup() + self.harness = Harness(self.setup) + self.client = self.harness.client + self.backend = MemoryLio() + self.mirror = LioMirror(self.setup.fake, self.setup.config, self.setup.credential, + self.backend, transport=FakeTransport(self.setup.fake), + session_kwargs={"sleep": lambda s: None}) + groups = self.setup.initiators[Protocol.ISCSI] + self.iqn_a, self.iqn_b = groups[0][0].address.lower(), groups[1][0].address.lower() + + def host(self, name: str, iqn: str) -> str: + return self.client.ensure_host(HostSpec( + name=name, initiators=(Initiator(protocol=Protocol.ISCSI, address=iqn),))) + + +@pytest.fixture(params=ISCSI_VENDORS) +def bed(request) -> Bed: + b = Bed(request.param) + yield b + b.client.close() + b.mirror.close() + + +def test_volumes_hosts_and_mappings_reach_the_target(bed): + vol = bed.client.create_volume(UUID, 10 * GIB) + host_a = bed.host("sbtest-a", bed.iqn_a) + bed.mirror.reconcile() + lun = bed.backend.volumes[bed.mirror.key(vol.id)] + assert lun.size_bytes == vol.size_bytes + # The host sees the identity the plug-in reports for the LUN. + assert lun.serial == host_wwn(vol, Protocol.ISCSI) + # A registered host can log in, and sees nothing yet. + assert bed.backend.acls[bed.iqn_a] == {} + + bed.client.map_volume(vol.id, host_a) + bed.mirror.reconcile() + (mapped,) = bed.backend.acls[bed.iqn_a].values() + assert mapped == bed.mirror.key(vol.id) + + # Multi-attach: a second host sees the same LUN. + host_b = bed.host("sbtest-b", bed.iqn_b) + bed.client.map_volume(vol.id, host_b) + bed.mirror.reconcile() + assert list(bed.backend.acls[bed.iqn_b].values()) == [bed.mirror.key(vol.id)] + + # Fence: the unmapped host loses the LUN, the other keeps it. + bed.client.unmap_volume(vol.id, host_a) + bed.mirror.reconcile() + assert bed.backend.acls[bed.iqn_a] == {} + assert list(bed.backend.acls[bed.iqn_b].values()) == [bed.mirror.key(vol.id)] + + grown = bed.client.resize_volume(vol.id, 20 * GIB) + bed.mirror.reconcile() + assert bed.backend.volumes[bed.mirror.key(vol.id)].size_bytes == grown.size_bytes + + bed.client.delete_volume(vol.id) + bed.mirror.reconcile() + assert bed.mirror.key(vol.id) not in bed.backend.volumes + assert bed.backend.acls[bed.iqn_b] == {} + + +def test_reconcile_is_idempotent(bed): + vol = bed.client.create_volume(UUID, 10 * GIB) + bed.client.map_volume(vol.id, bed.host("sbtest-a", bed.iqn_a)) + bed.mirror.reconcile() + before = bed.backend.ops + bed.mirror.reconcile() + assert bed.backend.ops == before + + +def test_allowed_initiators_can_log_in_before_they_are_registered(): + b = Bed("pure-flasharray") + b.mirror.allow = ["iqn.2026-01.io.sb:not-registered"] + b.mirror.reconcile() + assert b.backend.acls["iqn.2026-01.io.sb:not-registered"] == {} + + +def test_lun_numbers_follow_the_array(): + b = Bed("infinidat-infinibox") # reports a LUN number per mapping + vol = b.client.create_volume(UUID, 10 * GIB) + m = b.client.map_volume(vol.id, b.host("sbtest-a", b.iqn_a)) + b.mirror.reconcile() + assert list(b.backend.acls[b.iqn_a]) == [m.lun] diff --git a/tests/unit/storage_arrays/test_netapp_ontap.py b/tests/unit/storage_arrays/test_netapp_ontap.py new file mode 100644 index 0000000000..2c17efad9f --- /dev/null +++ b/tests/unit/storage_arrays/test_netapp_ontap.py @@ -0,0 +1,526 @@ +"""NetApp ONTAP adapter specifics beyond the shared contract.""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.adapters.netapp_ontap import ( + VOLUME_SUBSYSTEM_COMMENT, + NetAppOntapAdapter, + flexvol_name, + ontap_wwn, + parse_nsid, +) +from simplyblock_core.storage_arrays.base import ( + ArrayConflictError, + ArrayError, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeError, FakeTransport +from simplyblock_core.storage_arrays.testing.netapp_ontap import ( + FakeNetAppOntap, + contract_setup, +) + +from .conftest import StaticProvider + +GIB = 1 << 30 +NQN_A = "nqn.2014-08.org.nvmexpress:uuid:host-a" +NQN_B = "nqn.2014-08.org.nvmexpress:uuid:host-b" + + +def make(**options): + setup = contract_setup(**options) + client = ArrayClient(setup.config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + return client, setup.fake + + +def iscsi(name, *iqns): + return HostSpec(name=name, initiators=tuple( + Initiator(protocol=Protocol.ISCSI, address=a) for a in iqns)) + + +def nvme(name, *nqns): + return HostSpec(name=name, initiators=tuple( + Initiator(protocol=Protocol.NVME_TCP, address=a) for a in nqns)) + + +def calls(fake: FakeNetAppOntap, method: str, fragment: str) -> list[int]: + return [i for i, (m, p) in enumerate(fake.requests) if m == method and fragment in p] + + +# -- identity helpers ----------------------------------------------------------- + + +def test_wwn_from_serial(): + # 12 ASCII chars -> 24 hex digits behind NetApp's NAA-6 prefix + assert ontap_wwn("wCVt1]IlvQWv") == "naa.600a0980" + "7743567431" + "5d496c76515776" + assert len(ontap_wwn("wCVt1]IlvQWv")) == len("naa.") + 32 + + +def test_volume_wwn_matches_fake_serial(): + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), GIB) + serial = fake.luns[vol.id]["serial_number"] + assert vol.wwn == ontap_wwn(serial) == vol.identifier + assert vol.nguid is None + + +def test_parse_nsid_and_flexvol_name(): + assert parse_nsid("00000001h") == 1 + assert parse_nsid("0000001Ah") == 26 + assert parse_nsid(3) == 3 + assert parse_nsid(None) is None + assert flexvol_name("sb-0123abcd") == "sb_0123abcd" + + +# -- layout ------------------------------------------------------------------------ + + +def test_create_makes_thin_flexvol_then_lun(): + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), 10 * GIB) + lun = fake.luns[vol.id] + flex = fake.volumes[lun["location"]["volume"]["uuid"]] + assert flex["name"] == flexvol_name(vol.name) + assert lun["name"] == f"/vol/{flex['name']}/{vol.name}" + assert lun["os_type"] == "linux" + assert lun["space"]["guarantee"]["requested"] is False + assert flex["guarantee"]["type"] == "none" + assert flex["space"]["snapshot"]["reserve_percent"] == 0 + assert flex["snapshot_policy"]["name"] == "none" + assert flex["size"] >= 10 * GIB * 105 // 100 + assert calls(fake, "POST", "/api/storage/volumes")[0] < calls(fake, "POST", + "/api/storage/luns")[0] + + +def test_resize_grows_flexvol_before_lun(): + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), 4 * GIB) + fake.requests.clear() + grown = client.resize_volume(vol.id, 40 * GIB) + assert grown.size_bytes == 40 * GIB + vol_patch = calls(fake, "PATCH", "/api/storage/volumes/") + lun_patch = calls(fake, "PATCH", "/api/storage/luns/") + assert vol_patch and lun_patch and vol_patch[0] < lun_patch[0] + flex_uuid = fake.luns[vol.id]["location"]["volume"]["uuid"] + assert fake.volumes[flex_uuid]["size"] >= 40 * GIB + + +def test_fake_refuses_lun_larger_than_flexvol(): + """Guards the ordering above: the fake rejects a LUN that outgrows its FlexVol.""" + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), 4 * GIB) + adapter = client._adapter + assert adapter is not None + with pytest.raises(ArrayRequestError, match="exceeds"): + adapter._call("PATCH", f"/api/storage/luns/{vol.id}", + json={"space": {"size": 64 * GIB}}) + + +def test_delete_removes_lun_then_flexvol(): + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), GIB) + fake.requests.clear() + assert client.delete_volume(vol.id) + assert calls(fake, "DELETE", "/api/storage/luns/")[0] < calls( + fake, "DELETE", "/api/storage/volumes/")[0] + assert not fake.luns and not fake.volumes + + +def test_shared_flexvol_mode(): + client, fake = make(volume_per_lun=False, flexvol="sb_shared") + fake.add_flexvol("sb_shared", 100 * GIB) + vol = client.create_volume(str(uuid.uuid4()), 4 * GIB) + assert fake.luns[vol.id]["name"] == f"/vol/sb_shared/{vol.name}" + assert len(fake.volumes) == 1 + client.resize_volume(vol.id, 8 * GIB) + assert not calls(fake, "PATCH", "/api/storage/volumes/") + client.delete_volume(vol.id) + assert not fake.luns and [v["name"] for v in fake.volumes.values()] == ["sb_shared"] + + +def test_rerun_reuses_flexvol_of_interrupted_create(): + client, fake = make() + vol_uuid = str(uuid.uuid4()) + name = client.volume_name(vol_uuid) + fake.add_flexvol(flexvol_name(name), 2 * GIB) + vol = client.create_volume(vol_uuid, 8 * GIB) + assert len(fake.volumes) == 1 + assert fake.volumes[fake.luns[vol.id]["location"]["volume"]["uuid"]]["size"] >= 8 * GIB + + +def test_multiple_aggregates_pick_most_free_space(): + client, fake = make(aggregates="aggr1, aggr2") + vol = client.create_volume(str(uuid.uuid4()), GIB) + flex = fake.volumes[fake.luns[vol.id]["location"]["volume"]["uuid"]] + assert [a["name"] for a in flex["aggregates"]] == ["aggr2"] + + +# -- jobs ---------------------------------------------------------------------------- + + +def test_job_is_polled_until_success(): + client, fake = make() + fake.job_polls = 4 + client.create_volume(str(uuid.uuid4()), GIB) + assert len(calls(fake, "GET", "/api/cluster/jobs/")) == 5 + + +def test_job_failure_raises_and_rolls_nothing_back(): + client, fake = make() + fake.fail_next_job("Insufficient space in aggregate aggr1.", "917830") + with pytest.raises(ArrayRequestError, match="Insufficient space"): + client.create_volume(str(uuid.uuid4()), GIB) + assert not fake.volumes and not fake.luns + + +def test_duplicate_flexvol_job_failure_is_conflict(): + client, fake = make() + adapter_vol = "sb_dup" + fake.add_flexvol(adapter_vol, GIB) + client.get_info() + adapter = client._adapter + assert adapter is not None + with pytest.raises(ArrayConflictError, match="Duplicate"): + adapter._create_flexvol(adapter_vol, GIB) + + +def test_job_timeout(): + client, fake = make(job_timeout_s=10) + fake.jobs_stuck = True + with pytest.raises(ArrayRequestError, match="still running"): + client.create_volume(str(uuid.uuid4()), GIB) + + +def test_lun_create_failure_removes_new_flexvol(monkeypatch): + client, fake = make() + + def refuse(*a, **kw): + raise FakeError(400, {"error": {"message": "LUN creation refused", "code": "2"}}) + + monkeypatch.setattr(fake, "_check_size", refuse) + with pytest.raises(ArrayRequestError, match="refused"): + client.create_volume(str(uuid.uuid4()), GIB) + assert not fake.volumes + + +# -- hosts ------------------------------------------------------------------------------- + + +def test_igroup_is_mixed_linux_and_fc_wwpn_is_colon_lowercase(): + client, fake = make() + spec = HostSpec(name="h1", initiators=( + Initiator(protocol=Protocol.ISCSI, address="IQN.2026-01.io.sb:h1"), + Initiator(protocol=Protocol.FC, address="0x21000024FF4C1A2B"))) + assert client.ensure_host(spec) == "h1" + (group,) = fake.igroups.values() + assert group["protocol"] == "mixed" and group["os_type"] == "linux" + assert {i["name"] for i in group["initiators"]} == { + "iqn.2026-01.io.sb:h1", "21:00:00:24:ff:4c:1a:2b"} + + +def test_initiator_in_foreign_igroup_conflicts_although_ontap_allows_it(): + client, fake = make() + client.get_info() + adapter = client._adapter + assert adapter is not None + # ONTAP itself accepts the same IQN in a second igroup ... + for name in ("customer", "other"): + adapter._call("POST", "/api/protocols/san/igroups", json={ + "svm": {"name": "svm1"}, "name": name, "os_type": "linux", "protocol": "iscsi", + "initiators": [{"name": "iqn.2026-01.io.sb:x"}]}) + # ... the adapter refuses + with pytest.raises(ArrayConflictError, match="customer"): + client.ensure_host(iscsi("mine", "iqn.2026-01.io.sb:x")) + assert "mine" not in {g["name"] for g in fake.igroups.values()} + + +def test_existing_iscsi_igroup_cannot_take_fc(): + """ONTAP spec: an igroup's protocol cannot be changed after creation.""" + client, fake = make() + client.get_info() + adapter = client._adapter + assert adapter is not None + adapter._call("POST", "/api/protocols/san/igroups", json={ + "svm": {"name": "svm1"}, "name": "h2", "os_type": "linux", "protocol": "iscsi", + "initiators": [{"name": "iqn.2026-01.io.sb:h2"}]}) + with pytest.raises(ArrayConflictError, match="cannot take fcp"): + client.ensure_host(HostSpec(name="h2", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:h2"), + Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:99")))) + assert not calls(fake, "PATCH", "/api/protocols/san/igroups/") + # more iSCSI initiators still fit + client.ensure_host(iscsi("h2", "iqn.2026-01.io.sb:h2", "iqn.2026-01.io.sb:h2b")) + (group,) = fake.igroups.values() + assert group["protocol"] == "iscsi" and len(group["initiators"]) == 2 + + +def test_fake_refuses_igroup_protocol_change(): + client, fake = make() + client.ensure_host(iscsi("h3", "iqn.2026-01.io.sb:h3")) + adapter = client._adapter + assert adapter is not None + (group,) = fake.igroups.values() + group["protocol"] = "iscsi" + with pytest.raises(ArrayRequestError, match="cannot be changed"): + adapter._call("PATCH", f"/api/protocols/san/igroups/{group['uuid']}", + json={"protocol": "mixed"}) + + +def test_nvme_fc_wwpn_address_is_refused(): + client, _ = make() + with pytest.raises(ArrayUnsupportedError, match="NQN"): + client.ensure_host(HostSpec(name="fc", initiators=( + Initiator(protocol=Protocol.NVME_FC, address="21:00:00:24:ff:4c:1a:2b"),))) + + +def test_nvme_rdma_is_not_supported(): + client, _ = make() + with pytest.raises(ArrayError, match="does not support"): + client.ensure_host(HostSpec(name="r", initiators=( + Initiator(protocol=Protocol.NVME_RDMA, address=NQN_A),))) + + +# -- options ------------------------------------------------------------------------------- + + +@pytest.mark.parametrize(("options", "match"), [ + ({"svm": None}, "needs options: svm"), + ({"protocol_family": "fcoe"}, "protocol_family"), + ({"volume_per_lun": False}, "flexvol is required"), + ({"space_guarantee": "file"}, "space_guarantee"), + ({"snapshot_reserve_percent": 95}, "snapshot_reserve_percent"), + ({"namespace_block_size": 1024}, "namespace_block_size"), + ({"job_timeout_s": "soon"}, "job_timeout_s"), + ({"pool": "x"}, "Unknown NetApp ONTAP options"), +]) +def test_option_validation(options, match): + setup = contract_setup() + opts = {**setup.config.options, **options} + opts = {k: v for k, v in opts.items() if v is not None} + from simplyblock_core.storage_arrays.base import AdapterContext + from simplyblock_core.storage_arrays.http import RestSession + config = setup.config.model_copy(update={"options": opts}) + with pytest.raises(ArrayError, match=match): + NetAppOntapAdapter(AdapterContext(config, setup.credential, + RestSession(config.endpoint))) + + +def test_unknown_svm_fails_login(): + setup = contract_setup() + config = setup.config.model_copy(update={"options": {"svm": "nope"}}) + client = ArrayClient(config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake)) + with pytest.raises(ArrayNotFoundError, match="nope"): + client.get_info() + + +def test_get_info(): + client, _ = make() + info = client.get_info() + assert info.model == "AFF-A400" + assert info.version.startswith("NetApp Release 9.14.1") + assert info.serial + + +# -- NVMe family and protocol routing ------------------------------------------------------ + + +def test_nvme_family_creates_namespace(): + client, fake = make(protocol_family="nvme") + vol = client.create_volume(str(uuid.uuid4()), 8 * GIB) + assert vol.id in fake.namespaces and not fake.luns + ns = fake.namespaces[vol.id] + assert ns["space"]["block_size"] == 4096 and ns["os_type"] == "linux" + assert vol.wwn is None and vol.identifier == f"id.{vol.id}" + assert client.get_volume(vol.id) == vol + grown = client.resize_volume(vol.id, 16 * GIB, expected_identifier=vol.identifier) + assert grown.size_bytes == 16 * GIB + + +def test_nvme_multi_attach_through_volume_subsystem(): + client, fake = make(protocol_family="nvme") + vol = client.create_volume(str(uuid.uuid4()), 8 * GIB) + ha = client.ensure_host(nvme("node-a", NQN_A)) + hb = client.ensure_host(nvme("node-b", NQN_B)) + ma = client.map_volume(vol.id, ha) + assert ma.lun == 1 and ma.host_id == "node-a" + assert client.map_volume(vol.id, ha) == ma + client.map_volume(vol.id, hb) + assert {m.host_id for m in client.list_mappings(vol.id)} == {"node-a", "node-b"} + (smap,) = fake.subsystem_maps + vsub = fake.subsystems[smap["subsystem"]["uuid"]] + assert vsub["comment"] == VOLUME_SUBSYSTEM_COMMENT + assert vsub["name"] == flexvol_name(vol.name) + assert {h["nqn"] for h in vsub["hosts"]} == {NQN_A, NQN_B} + + client.unmap_volume(vol.id, ha) + client.unmap_volume(vol.id, ha) + assert [m.host_id for m in client.list_mappings(vol.id)] == ["node-b"] + + assert client.delete_volume(vol.id) + assert not fake.namespaces and not fake.subsystem_maps and not fake.volumes + assert {s["name"] for s in fake.subsystems.values()} == {"node-a", "node-b"} + client.delete_host(ha) + client.delete_host(hb) + assert not fake.subsystems + + +def test_volume_subsystem_does_not_count_as_host_owner(): + client, _ = make(protocol_family="nvme") + vol = client.create_volume(str(uuid.uuid4()), GIB) + client.map_volume(vol.id, client.ensure_host(nvme("node-a", NQN_A))) + # NQN_A is now in node-a and in the volume subsystem: re-ensuring is fine, + assert client.ensure_host(nvme("node-a", NQN_A)) == "node-a" + # but another host claiming it conflicts. + with pytest.raises(ArrayConflictError, match="node-a"): + client.ensure_host(nvme("node-x", NQN_A)) + + +def test_scsi_volume_to_nvme_host_is_unsupported_and_vice_versa(): + client, _ = make() + lun = client.create_volume(str(uuid.uuid4()), GIB) + host = client.ensure_host(nvme("nv", NQN_A)) + with pytest.raises(ArrayUnsupportedError, match="NVMe host"): + client.map_volume(lun.id, host) + + nclient, nfake = make(protocol_family="nvme") + ns = nclient.create_volume(str(uuid.uuid4()), GIB) + nclient.ensure_host(iscsi("sc", "iqn.2026-01.io.sb:sc")) + with pytest.raises(ArrayUnsupportedError, match="SCSI host"): + nclient.map_volume(ns.id, "sc") + assert not nfake.subsystem_maps + + +def test_volumes_of_the_other_family_are_still_found(): + client, fake = make(protocol_family="nvme") + ns = client.create_volume(str(uuid.uuid4()), GIB) + scsi_setup_config = client.config.model_copy( + update={"options": {"svm": "svm1", "protocol_family": "scsi"}}) + provider = StaticProvider(contract_setup().credential) + scsi_client = ArrayClient(scsi_setup_config, provider, # type: ignore[arg-type] + transport=FakeTransport(fake)) + assert scsi_client.get_volume(ns.id) == ns + assert scsi_client.delete_volume(ns.id) + assert not fake.namespaces + + +def test_namespace_mapped_to_foreign_subsystem_is_refused(): + client, fake = make(protocol_family="nvme") + vol = client.create_volume(str(uuid.uuid4()), GIB) + adapter = client._adapter + assert adapter is not None + adapter._call("POST", "/api/protocols/nvme/subsystems", json={ + "svm": {"name": "svm1"}, "name": "customer_ss", "os_type": "linux"}) + adapter._call("POST", "/api/protocols/nvme/subsystem-maps", json={ + "svm": {"name": "svm1"}, "namespace": {"uuid": vol.id}, + "subsystem": {"name": "customer_ss"}}) + client.ensure_host(nvme("node-a", NQN_A)) + with pytest.raises(ArrayConflictError, match="does not manage"): + client.map_volume(vol.id, "node-a") + + +def test_lun_numbers_come_from_lun_maps(): + client, fake = make() + host = client.ensure_host(iscsi("h", "iqn.2026-01.io.sb:h")) + v1 = client.create_volume(str(uuid.uuid4()), GIB) + v2 = client.create_volume(str(uuid.uuid4()), GIB) + assert client.map_volume(v1.id, host).lun == 0 + assert client.map_volume(v2.id, host).lun == 1 + assert {m["logical_unit_number"] for m in fake.lun_maps} == {0, 1} + + +def test_map_to_missing_host_is_not_found(): + client, _ = make() + vol = client.create_volume(str(uuid.uuid4()), GIB) + with pytest.raises(ArrayNotFoundError): + client.map_volume(vol.id, "ghost") + + +# -- behaviour checked against vendor source ---------------------------------------------- + + +@pytest.mark.parametrize("family", ["scsi", "nvme"]) +def test_lun_and_namespace_calls_may_be_jobs(family): + """The ONTAP spec allows 202 + job for LUN/namespace create/modify/delete.""" + client, fake = make(protocol_family=family) + fake.async_san = True + fake.job_polls = 2 + vol = client.create_volume(str(uuid.uuid4()), 4 * GIB) + assert client.get_volume(vol.id) == vol + assert client.resize_volume(vol.id, 8 * GIB).size_bytes == 8 * GIB + host = client.ensure_host(nvme("n", NQN_A) if family == "nvme" + else iscsi("s", "iqn.2026-01.io.sb:s")) + client.map_volume(vol.id, host) + client.unmap_volume(vol.id, host) + assert client.delete_volume(vol.id) + assert not fake.luns and not fake.namespaces and not fake.volumes + san = "/api/storage/luns" if family == "scsi" else "/api/storage/namespaces" + assert {fake.jobs[j]["description"].split(" ")[0] for j in fake.jobs + if san in fake.jobs[j]["description"]} == {"POST", "PATCH", "DELETE"} + + +def test_async_lun_create_failure_is_classified(): + client, fake = make() + fake.async_san = True + vol = client.create_volume(str(uuid.uuid4()), GIB) + adapter = client._adapter + assert adapter is not None + with pytest.raises(ArrayConflictError, match="5374242"): + adapter._async("POST", "/api/storage/luns", json={ + "svm": {"name": "svm1"}, "name": fake.luns[vol.id]["name"], "os_type": "linux", + "space": {"size": GIB}}) + + +def test_deleted_flexvol_goes_to_recovery_queue_by_default(): + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), GIB) + client.delete_volume(vol.id) + assert not fake.volumes and len(fake.recovery_queue) == 1 + + +def test_skip_recovery_queue_deletes_with_force(): + client, fake = make(skip_recovery_queue=True) + vol = client.create_volume(str(uuid.uuid4()), GIB) + client.delete_volume(vol.id) + assert not fake.volumes and not fake.recovery_queue + + +def test_asa_r2_is_refused_at_login(): + client, fake = make() + fake.version = (9, 16, 1) + fake.disaggregated = True + with pytest.raises(ArrayUnsupportedError, match="ASA r2"): + client.get_info() + assert not fake.volumes + + +def test_unified_9_16_is_accepted(): + client, fake = make() + fake.version = (9, 16, 1) + assert client.get_info().version.startswith("NetApp Release 9.16.1") + assert any(m == "GET" and p == "/api/cluster" for m, p in fake.requests) + + +@pytest.mark.parametrize(("code", "message", "exc"), [ + ("4", "entry missing", ArrayNotFoundError), + ("1", "entry exists", ArrayConflictError), + ("5374242", "LUN exists.", ArrayConflictError), + ("72090025", "subsystem exists", ArrayConflictError), + ("5374922", "map exists", ArrayConflictError), + ("917627", "something else", ArrayRequestError), +]) +def test_error_codes_are_classified(code, message, exc): + err = NetAppOntapAdapter._error("POST", "/api/x", {"error": {"message": message, + "code": code}}) + assert type(err) is exc diff --git a/tests/unit/storage_arrays/test_pure.py b/tests/unit/storage_arrays/test_pure.py new file mode 100644 index 0000000000..9f5831b30c --- /dev/null +++ b/tests/unit/storage_arrays/test_pure.py @@ -0,0 +1,130 @@ +"""Pure FlashArray adapter specifics beyond the shared contract.""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.adapters.pure import ( + PureFlashArrayAdapter, + pure_nguid, + pure_wwn, +) +from simplyblock_core.storage_arrays.base import ( + ArrayConflictError, + ArrayNotFoundError, + ArrayRequestError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeTransport +from simplyblock_core.storage_arrays.testing.pure import contract_setup + +from .conftest import StaticProvider + +GIB = 1 << 30 + + +def make(**options): + setup = contract_setup() + config = setup.config.model_copy(update={"options": {**setup.config.options, **options}}) + client = ArrayClient(config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + return client, setup.fake + + +def test_identity_matches_vendor_examples(): + # Sample values from Ansible purestorage.flasharray purefa_volume.py RETURN docs. + assert pure_nguid("CD6B99EF258647C5000BE684") == "eui.00cd6b99ef25864724a937c5000be684" + assert pure_wwn("361019ECACE43DB3000120A4") == "naa.624a9370361019ecace43db3000120a4" + + +def test_newest_supported_rest_version_is_used(): + client, fake = make() + client.get_info() + # The fake offers 1.19 .. 2.40; 2.40 is above what the adapter was written for. + assert ("POST", "/api/2.36/login") in fake.requests + assert all(not p.startswith("/api/2.40") for _, p in fake.requests) + + +def test_pinned_api_version(): + client, fake = make(api_version="2.21") + client.get_info() + assert ("POST", "/api/2.21/login") in fake.requests + assert ("GET", "/api/api_version") not in fake.requests + + +def test_no_usable_version(): + client, fake = make() + fake.versions = ["1.19", "2.0", "2.2"] + with pytest.raises(ArrayRequestError, match="no REST 2.x version"): + client.get_info() + + +def test_nvme_connection_reports_nsid(): + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), GIB) + host = client.ensure_host(HostSpec(name="nv", initiators=( + Initiator(protocol=Protocol.NVME_TCP, address="nqn.2014-08.org.nvmexpress:uuid:nv"),))) + m = client.map_volume(vol.id, host) + (conn,) = fake.connections + assert "nsid" in conn and "lun" not in conn + assert m.lun == conn["nsid"] == 1 + assert client.list_mappings(vol.id) == [m] + + +def test_scsi_connection_reports_lun(): + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), GIB) + host = client.ensure_host(HostSpec(name="sc", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:sc"),))) + assert client.map_volume(vol.id, host).lun == fake.connections[0]["lun"] == 1 + + +def test_fc_wwns_are_sent_upper_case_without_colons(): + client, fake = make() + client.ensure_host(HostSpec(name="fc", initiators=( + Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),))) + assert fake.hosts["fc"]["wwns"] == ["21000024FF4C1A2B"] + + +def test_existing_host_gets_add_fields(): + client, fake = make() + client.ensure_host(HostSpec(name="h", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:a"),))) + client.ensure_host(HostSpec(name="h", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:a"), + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:b")))) + assert fake.hosts["h"]["iqns"] == ["iqn.2026-01.io.sb:a", "iqn.2026-01.io.sb:b"] + assert ("PATCH", "/api/2.36/hosts") in fake.requests + + +def test_eradicate_option(): + client, fake = make(eradicate=True) + vol = client.create_volume(str(uuid.uuid4()), GIB) + client.delete_volume(vol.id) + assert not fake.volumes + + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), GIB) + client.delete_volume(vol.id) + assert fake.volumes[vol.id]["destroyed"] is True + assert client.get_volume(vol.id) is None + + +@pytest.mark.parametrize(("message", "exc"), [ + ("Volume does not exist.", ArrayNotFoundError), + ("Host already exists.", ArrayConflictError), + ("Implicit truncation is not permitted.", ArrayRequestError), +]) +def test_error_messages_are_classified(message, exc): + client, fake = make() + client.get_info() + adapter = client._adapter + assert isinstance(adapter, PureFlashArrayAdapter) + fake.fail_next(400, {"errors": [{"message": message, "context": "x"}]}) + with pytest.raises(exc): + adapter._items("GET", "/volumes", params={"names": "x"}) diff --git a/tests/unit/storage_arrays/test_san_plugin.py b/tests/unit/storage_arrays/test_san_plugin.py new file mode 100644 index 0000000000..87e7836b38 --- /dev/null +++ b/tests/unit/storage_arrays/test_san_plugin.py @@ -0,0 +1,168 @@ +"""The SAN array plugins (simplyblock_core.array.san): every storage_arrays +adapter driven through the multi-attach ArrayPlugin contract, against its +vendor fake.""" +from __future__ import annotations + +import pytest + +from simplyblock_core import array, multi_attach +from simplyblock_core.array import san +from simplyblock_core.storage_arrays.base import Protocol +from simplyblock_core.storage_arrays.testing import FakeTransport + +from .conftest import VENDORS, fake_module + +GIB = 1 << 30 +LUN = "simplyblock-node-a-s0" + +_KEY = { + Protocol.ISCSI: "iscsi_iqn", + Protocol.FC: "fc_wwpns", + Protocol.NVME_TCP: "nvme_hostnqn", + Protocol.NVME_RDMA: "nvme_hostnqn", + Protocol.NVME_FC: "nvme_hostnqn", + Protocol.SDC: "sdc_guid", +} + + +class Bed: + def __init__(self, vendor: str): + self.vendor = vendor + self.setup = fake_module(vendor).contract_setup() + protos = list(self.setup.initiators) + self.protocol = Protocol.ISCSI if Protocol.ISCSI in protos else protos[0] + self.creds = {n: self.setup.credential.get(n) for n in self.setup.credential.field_names} + + def plugin(self, cluster_id: str = "cluster-1", **extra) -> san.SanArrayPlugin: + options = {**self.setup.config.options, "protocol": self.protocol.value, **extra} + p = array.plugin_class(self.vendor)(endpoint=self.setup.config.endpoint, options=options, + cluster_id=cluster_id) + assert isinstance(p, san.SanArrayPlugin) + p.transport = FakeTransport(self.setup.fake) + p.session_kwargs = {"sleep": lambda s: None} + p.connect(self.setup.config.endpoint, self.creds, options) + return p + + def initiators(self, which: int) -> dict: + group = self.setup.initiators[self.protocol][which] + key = _KEY[self.protocol] + addresses = [i.address for i in group] + value = addresses if key == "fc_wwpns" else addresses[0] + return {"hostname": f"node-{which}", key: value} + + +@pytest.fixture(params=VENDORS) +def bed(request) -> Bed: + return Bed(request.param) + + +def test_registered_as_builtin_plugins(): + assert set(san.SAN_PLUGINS) == set(VENDORS) + for name, path in san.SAN_PLUGINS.items(): + assert array.BUILTIN_PLUGINS[name] == path + assert array.plugin_class(name).vendor == name + + +def test_lifecycle_multi_attach_fence_grow_delete(bed): + p = bed.plugin() + caps = p.capabilities() + assert bed.protocol.value in caps["protocols"] + host_a = p.ensure_host("node-a", bed.initiators(0)) + assert p.ensure_host("node-a", bed.initiators(0)) == host_a # idempotent + + lun = p.create_lun(LUN, 10 * GIB, {"simplyblock-host": "node-a"}) + assert lun["size"] >= 10 * GIB + assert lun["wwn"] and not lun["wwn"].startswith("naa.") + assert p.create_lun(LUN, 10 * GIB, {})["id"] == lun["id"] # retried create + + p.attach(lun["id"], host_a) + p.attach(lun["id"], host_a) # idempotent + view = p.get(lun["id"]) + assert view["hosts"] == [host_a] + assert multi_attach.normalize_wwn(view["wwn"]) == multi_attach.normalize_wwn(lun["wwn"]) + + host_b = p.ensure_host("node-b", bed.initiators(1)) + p.attach(lun["id"], host_b) + assert sorted(p.get(lun["id"])["hosts"]) == sorted([host_a, host_b]) + p.detach(lun["id"], host_b) + assert p.get(lun["id"])["hosts"] == [host_a] + + assert p.fence(lun["id"], host_a) + assert p.get(lun["id"])["hosts"] == [] + assert p.unfence(lun["id"], host_a) + assert p.get(lun["id"])["hosts"] == [host_a] + + p.resize(lun["id"], 20 * GIB) + assert p.get(lun["id"])["size"] >= 20 * GIB + + p.delete_lun(lun["id"]) + with pytest.raises(array.ArrayError): + p.get(lun["id"]) + p.delete_lun(lun["id"]) # gone: succeeds + p.detach(lun["id"], host_a) # gone: succeeds + + +def test_other_clusters_luns_are_never_touched(bed): + mine = bed.plugin("cluster-1") + theirs = bed.plugin("cluster-2") + lun = mine.create_lun(LUN, 10 * GIB, {}) + # Same requested name, other cluster: a different LUN. + assert theirs.create_lun(LUN, 10 * GIB, {})["id"] != lun["id"] + for op in (lambda: theirs.get(lun["id"]), lambda: theirs.delete_lun(lun["id"]), + lambda: theirs.resize(lun["id"], 20 * GIB), + lambda: theirs.attach(lun["id"], "h"), lambda: theirs.detach(lun["id"], "h")): + with pytest.raises(array.ForeignLunError): + op() + assert mine.get(lun["id"])["size"] >= 10 * GIB + + foreign = bed.setup.create_foreign_volume("prod-db-01", 5 * GIB) + with pytest.raises(array.ForeignLunError): + mine.delete_lun(foreign) + + +def test_connect_validation(bed): + with pytest.raises(array.ArrayError, match="protocol"): + bed.plugin(protocol="") + with pytest.raises(array.ArrayError, match="unknown protocol"): + bed.plugin(protocol="carrier-pigeon") + unsupported = [p for p in Protocol if p.value not in + bed.plugin().capabilities()["protocols"]] + if unsupported: + with pytest.raises(array.ArrayError, match="does not support"): + bed.plugin(protocol=unsupported[0].value) + + +def test_host_without_initiator_of_the_protocol(bed): + p = bed.plugin() + with pytest.raises(array.ArrayError, match="reports no"): + p.ensure_host("node-a", {"hostname": "node-a"}) + + +def test_list_luns_falls_back_to_recorded(bed): + with pytest.raises(NotImplementedError): + bed.plugin().list_luns() + + +def test_names(): + assert len(san.lun_uuid("c", "x")) == 36 + a, b = san.lun_uuid("c1", LUN), san.lun_uuid("c2", LUN) + assert a != b and a == san.lun_uuid("c1", LUN) + assert san.host_object_name("node-a.example.com", 63) == "sb-node-a-example-com" + long = san.host_object_name("x" * 80, 31) + assert len(long) == 31 and long.startswith("sb-x") + assert san.host_object_name("x" * 80, 31) == long + assert san.host_object_name("y" * 80, 31) != long + + +# NetApp creates SCSI LUNs unless told otherwise; an NVMe protocol without +# protocol_family would create LUNs the NVMe hosts never see. +def test_netapp_protocol_family_follows_the_protocol(): + bed = Bed("netapp-ontap") + bed.protocol = Protocol.NVME_TCP + assert bed.plugin().client.config.options["protocol_family"] == "nvme" + bed.protocol = Protocol.ISCSI + assert bed.plugin().client.config.options.get("protocol_family", "scsi") == "scsi" + # A stated family that contradicts the protocol is refused. + bed.protocol = Protocol.NVME_TCP + with pytest.raises(array.ArrayError, match="protocol_family"): + bed.plugin(protocol_family="scsi") diff --git a/tests/unit/test_activation_growth_guard.py b/tests/unit/test_activation_growth_guard.py index 33a3b88d1a..f57fc293d0 100644 --- a/tests/unit/test_activation_growth_guard.py +++ b/tests/unit/test_activation_growth_guard.py @@ -21,6 +21,10 @@ def _cluster(status=Cluster.STATUS_SUSPENDED, activated=()): cluster = MagicMock() cluster.status = status cluster.activated_node_ids = list(activated) + # Model defaults: as child mocks these read truthy and steer activation + # into the sync-replication / failure-domain branches. + cluster.sync_replication = False + cluster.enable_failure_domain = False cluster.get_id.return_value = "cl-1" return cluster @@ -37,9 +41,12 @@ def _nodes(*ids): class TestActivationRefusesAddedNodes(unittest.TestCase): def _activate(self, cluster, nodes): - """Run only far enough to hit (or pass) the growth guard.""" + """Run only far enough to hit (or pass) the growth guard. The resource + floors are not what these mock nodes model (test_cluster_floors covers + them), so they are patched out.""" with patch.object(cluster_ops, "db_controller") as db, \ patch.object(cluster_ops, "set_cluster_status"), \ + patch.object(cluster_ops.floors, "evaluate", return_value=[]), \ patch.object(cluster_ops, "logger"): db.get_cluster_by_id.return_value = cluster db.get_storage_nodes_by_cluster_id.return_value = nodes diff --git a/tests/unit/test_array_journal.py b/tests/unit/test_array_journal.py new file mode 100644 index 0000000000..8a15b11395 --- /dev/null +++ b/tests/unit/test_array_journal.py @@ -0,0 +1,315 @@ +"""Where a node's journal is kept on a multi-attach cluster whose array plugin +creates the nodes' LUNs (Cluster.array_journal): + + - "lun": node add creates a journal LUN next to the data LUN, with the + journal options (a faster pool) over the array's options and its own + size, presented to that node only; the host config names no device + (sn configure --lblk-array-only). The LUN is recorded on the node, + detached with it and deleted with the cluster. + - "local": the host config names one local disk or partition + (sn configure --lblk-journal-only), used whole. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import utils +from simplyblock_core.array.fake import FakeArrayPlugin +from simplyblock_core.controllers import array_controller +from tests.unit import test_lblk_journal_disk_split as journal_split +from tests.unit.test_array_lifecycle import ( + CID, + GIB, + PLUGIN, + FakeDB, + _Case, + _cluster, + _lun_device, + _node, +) + + +def _journal_cluster(journal="lun", lun_size=100 * GIB, journal_lun_size=0, journal_options=None): + cl = _cluster(lun_size=lun_size) + cl.array_journal = journal + cl.array_journal_lun_size = journal_lun_size + cl.array_journal_options = dict(journal_options or {}) + return cl + + +class TestRegistration(_Case): + + def test_journal_lun_with_its_own_pool(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + view = array_controller.register_array( + CID, PLUGIN, options={"pool": "capacity"}, lun_size=100 * GIB, journal="lun", + journal_lun_size=8 * GIB, journal_options={"pool": "fast"}) + self.assertEqual((db.cluster.array_journal, db.cluster.array_journal_lun_size, + db.cluster.array_journal_options), ("lun", 8 * GIB, {"pool": "fast"})) + self.assertEqual((view["journal"], view["journal_lun_size"], view["journal_options"]), + ("lun", 8 * GIB, {"pool": "fast"})) + # The journal options were checked by the plugin, over the array's. + self.assertIn({"pool": "fast"}, [c[2] for c in FakeArrayPlugin.connected]) + # The array's own options are unchanged. + self.assertEqual(db.cluster.array_options, {"pool": "capacity"}) + + def test_default_is_local(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + view = array_controller.register_array(CID, PLUGIN, lun_size=GIB) + self.assertEqual((db.cluster.array_journal, view["journal"], view["journal_lun_size"]), + ("local", "local", 0)) + + def test_refusals(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + for kw, msg in (({"journal": "nvram"}, "journal must be one of"), + ({"journal": "lun"}, "needs lun_size"), + ({"lun_size": GIB, "journal_options": {"pool": "f"}}, "'lun' only"), + ({"lun_size": GIB, "journal_lun_size": GIB}, "'lun' only"), + ({"lun_size": GIB, "journal": "lun", "journal_lun_size": -1}, "negative")): + with self.subTest(kw=kw), self.assertRaisesRegex(ValueError, msg): + array_controller.register_array(CID, PLUGIN, **kw) + self.assertEqual(db.cluster.array_plugin, "") + + def test_bad_journal_options_save_nothing(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + real_connect = FakeArrayPlugin.connect + + def connect(self, endpoint, credentials, options): + if options.get("pool") == "missing": + raise array_controller.array.ArrayError("no pool 'missing'") + return real_connect(self, endpoint, credentials, options) + with patch.object(FakeArrayPlugin, "connect", connect), \ + self.assertRaisesRegex(ValueError, "journal options: .*no pool"): + array_controller.register_array(CID, PLUGIN, lun_size=GIB, journal="lun", + journal_options={"pool": "missing"}) + self.assertEqual(db.cluster.array_plugin, "") + + def test_delete_resets_and_counts_journal_luns(self): + a = _node("a") + db = self.use_db(FakeDB(_cluster(plugin=""), [a])) + array_controller.register_array(CID, PLUGIN, lun_size=GIB, journal="lun", + journal_options={"pool": "fast"}) + a.array_journal_lun_id = "lun-j" + with self.assertRaisesRegex(ValueError, "1 LUN"): + array_controller.delete_array(CID, db.cluster.array_id) + array_controller.delete_array(CID, db.cluster.array_id, force=True) + self.assertEqual((db.cluster.array_journal, db.cluster.array_journal_options), ("local", {})) + + +class TestLocalKeyStoreOptIn(_Case): + """The opt-in to the local key store travels with the registration, so an + operator-managed cluster never needs the CLI for it.""" + + def test_opt_in_with_the_registration(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + db.cluster.allow_local_kms_credentials = False + with self.assertRaisesRegex(ValueError, "could not be stored"): + array_controller.register_array(CID, PLUGIN, credentials={"p": "x"}) + self.assertFalse(db.cluster.allow_local_kms_credentials) + array_controller.register_array(CID, PLUGIN, credentials={"p": "x"}, allow_local_kms_credentials=True) + self.assertTrue(db.cluster.allow_local_kms_credentials) + self.assertEqual(db.cluster.array_credential_ref["kind"], "managed") + # Omitted: unchanged. + array_controller.register_array(CID, PLUGIN, credentials={"p": "y"}) + self.assertTrue(db.cluster.allow_local_kms_credentials) + + +class TestSize(unittest.TestCase): + + def test_default_is_3_percent_at_least_2_gib_whole_gib(self): + self.assertEqual(array_controller.journal_lun_size(_journal_cluster(lun_size=1024 * GIB)), + 31 * GIB) # 30.72 GiB rounded up + self.assertEqual(array_controller.journal_lun_size(_journal_cluster(lun_size=10 * GIB)), 2 * GIB) + self.assertEqual(array_controller.journal_lun_size( + _journal_cluster(journal_lun_size=5 * GIB + 1)), 5 * GIB + 1) + + def test_gate(self): + self.assertTrue(array_controller.journal_on_lun(_journal_cluster())) + self.assertFalse(array_controller.journal_on_lun(_journal_cluster(journal="local"))) + self.assertFalse(array_controller.journal_on_lun(_journal_cluster(lun_size=0))) + cl = _journal_cluster() + cl.array_journal = "" # a record from before the field + self.assertFalse(array_controller.journal_on_lun(cl)) + + +class TestProvisioning(_Case): + + def _api(self): + """Every LUN the fake array holds is a disk of the host.""" + api = MagicMock() + api.scan_new_luns.return_value = ({}, None) + + def inv(): + return [{"type": "disk", "name": f"sd{i}", "serial": lun["wwn"], "wwn": "", + "size": lun["size"], "by_id_path": f"/dev/disk/by-id/{lun['wwn']}", "numa_node": 0} + for i, lun in enumerate(FakeArrayPlugin.luns.values())], None + api.get_blockdevices.side_effect = inv + return api + + def test_journal_lun_then_data_lun(self): + cl = _journal_cluster(journal_options={"pool": "fast"}, journal_lun_size=4 * GIB) + cl.array_options = {"pool": "capacity", "thin": True} + api, info = self._api(), {"hostname": "w1"} + jentry, host = array_controller.provision_node_lun(cl, api, info, {"socket": 0}, [], journal=True) + cfg = array_controller.add_lun_to_config([], jentry) + dentry, _ = array_controller.provision_node_lun(cl, api, info, {"socket": 0}, cfg) + cfg = array_controller.add_lun_to_config(cfg, dentry) + + jlun = FakeArrayPlugin.luns[jentry["array_lun_id"]] + dlun = FakeArrayPlugin.luns[dentry["array_lun_id"]] + self.assertEqual((jlun["name"], jlun["size"], jlun["hosts"]), ("simplyblock-w1-s0-journal", 4 * GIB, [host])) + self.assertEqual(jlun["tags"]["simplyblock-role"], "journal") + self.assertEqual((dlun["name"], dlun["size"]), ("simplyblock-w1-s0", 100 * GIB)) + # The journal LUN was created through a plugin with the journal pool, + # the data LUN through one with the array's own options. + pools = [c[2].get("pool") for c in FakeArrayPlugin.connected] + self.assertEqual(pools, ["fast", "capacity"]) + self.assertEqual(FakeArrayPlugin.connected[0][2]["thin"], True) + # The journal entry is flagged, the data entry is not, and both stay. + self.assertEqual([bool(e.get("journal")) for e in cfg], [True, False]) + self.assertEqual(array_controller.journal_lun_id(cfg), jentry["array_lun_id"]) + + def test_retried_add_reuses_each_kind(self): + cl = _journal_cluster() + api, info = self._api(), {"hostname": "w1"} + jentry, _ = array_controller.provision_node_lun(cl, api, info, {"socket": 0}, [], journal=True) + dentry, _ = array_controller.provision_node_lun(cl, api, info, {"socket": 0}, [jentry]) + cfg = [jentry, dentry] + j2, _ = array_controller.provision_node_lun(cl, api, info, {"socket": 0}, cfg, journal=True) + d2, _ = array_controller.provision_node_lun(cl, api, info, {"socket": 0}, cfg) + self.assertEqual((j2["array_lun_id"], d2["array_lun_id"]), + (jentry["array_lun_id"], dentry["array_lun_id"])) + self.assertEqual(len(FakeArrayPlugin.luns), 2) + + def test_journal_lun_is_not_multi_attached(self): + from simplyblock_core import multi_attach + from simplyblock_core.models.nvme_device import NVMeDevice + cl = _journal_cluster() + api, info = self._api(), {"hostname": "w1"} + jentry, _ = array_controller.provision_node_lun(cl, api, info, {"socket": 0}, [], journal=True) + dev = NVMeDevice() + dev.serial_number = jentry["serial"] + multi_attach.mark_lun_devices(cl, _node("a"), [dev], [jentry]) + self.assertFalse(dev.multi_attach) + self.assertEqual(dev.array_lun_id, "") + + +class TestReleaseAndDelete(_Case): + + def test_node_release_detaches_its_journal_lun(self): + a = _node("a") + db = self.use_db(FakeDB(_journal_cluster(), [a])) + p = self.plugin(db.cluster) + data, journal = p.create_lun("d", GIB, {}), p.create_lun("j", GIB, {}) + for lun in (data, journal): + p.attach(lun["id"], "host-a") + _lun_device(a, data["id"], data["wwn"]) + a.array_journal_lun_id = journal["id"] + array_controller.release_node_luns(a) + self.assertEqual((p.get(data["id"])["hosts"], p.get(journal["id"])["hosts"]), ([], [])) + self.assertEqual(len(FakeArrayPlugin.luns), 2) # released, not deleted + + def test_cluster_delete_deletes_recorded_journal_luns(self): + a = _node("a") + cl = _journal_cluster() + self.use_db(FakeDB(cl, [a])) + p = self.plugin(cl) + journal = p.create_lun("j", GIB, {}) + a.array_journal_lun_id = journal["id"] + with patch.object(FakeArrayPlugin, "list_luns", side_effect=NotImplementedError): + array_controller.delete_cluster_luns(cl) + self.assertEqual(FakeArrayPlugin.luns, {}) + + +class TestHostConfig(unittest.TestCase): + """sn configure: --lblk-array-only (no local device) and a local journal + on a partition (--lblk-journal-only).""" + + def test_journal_only_accepts_a_partition(self): + out = utils.lblk_journal_only_layout( + {"nvme0n1p3": {"name": "nvme0n1p3", "serial": "P", "size": GIB, "type": "part"}}) + self.assertTrue(out["nvme0n1p3"]["journal"]) + + def test_array_only_node_counts_its_two_luns(self): + self.assertEqual(utils.node_config_device_count({"lblk_devices": [], "lblk_array_only": True}), 2) + self.assertEqual(utils.node_config_device_count({"lblk_devices": [], "ssd_pcis": []}), 0) + + def test_array_only_sys_memory_counts_the_luns(self): + node = {"lblk_devices": [], "lblk_array_only": True} + self.assertEqual(utils.node_config_min_sys_memory(node, 10 * 1024 * GIB), + int(utils.sys_memory_for_capacity(10 * 1024 * GIB))) + + def _node(self, **kw): + node = {"socket": 0, "cpu_mask": "0x2", "isolated": [1], "l-cores": "1@1", "ssd_pcis": [], + "lblk_devices": [], "distribution": {}} + node.update(kw) + return node + + def test_validate_array_only(self): + fields = {f: [] for f in ("app_thread_core", "jm_cpu_core", "poller_cpu_cores", "alceml_cpu_cores", + "distrib_cpu_cores", "jc_singleton_core")} + with patch.object(utils, "validate_node_config", wraps=utils.validate_node_config): + base = self._node(distribution=fields) + for k in ("max_lvol", "max_size", "huge_page_memory", "number_of_distribs", "nic_ports", + "number_of_alcemls", "small_pool_count", "large_pool_count", "sys_memory"): + base.setdefault(k, 1) + self.assertFalse(utils.validate_node_config(dict(base))) + self.assertTrue(utils.validate_node_config(dict(base, lblk_array_only=True))) + self.assertFalse(utils.validate_node_config(dict( + base, lblk_array_only=True, + lblk_devices=[{"name": "sdb", "serial": "S", "size": GIB}]))) + + +class TestGenerateArrayOnly(unittest.TestCase): + # The harness only; its tests stay in their module. + _run = journal_split.TestGenerateConfigsPerNode._run + + def test_array_only_node_has_no_device(self): + config, calls = self._run({}, nodes_per_socket=2, selection={"array_only": True}) + self.assertEqual(calls, []) + self.assertEqual([(n["lblk_devices"], n["lblk_array_only"]) for n in config["nodes"]], + [([], True), ([], True)]) + + +class TestConfigureArgs(unittest.TestCase): + + def _args(self, **kw): + import argparse + base = {"max_lvol": "10", "max_prov": "", "pci_allowed": "", "pci_blocked": "", "lblk": True, + "blk_names": "", "blk_names_exclude": "", "blk_serials": "", "lblk_journal_only": False, + "lblk_journal_device": False, "lblk_array_only": True, "device_model": "", "size_range": "", + "nvme_names": "", "upgrade": False} + base.update(kw) + return argparse.Namespace(**base) + + def test_node_configure_validation(self): + import argparse + + from simplyblock_web import node_configure + node_configure.validate_arguments(self._args()) + for kw in ({"lblk": False}, {"blk_names": "sdb"}, {"lblk_journal_only": True}, + {"lblk_journal_device": True}): + with self.subTest(kw=kw), self.assertRaises(argparse.ArgumentError): + node_configure.validate_arguments(self._args(**kw)) + + def test_cli_has_the_flags(self): + from simplyblock_cli import cli + with open(cli.__file__, encoding="utf-8") as f: + text = f.read() + for flag in ("'--lblk-array-only'", "'--journal'", "'--journal-lun-size'", "'--journal-options'"): + self.assertIn(flag, text) + + def test_cli_set_array_passes_the_journal(self): + from simplyblock_cli.clibase import CLIWrapperBase + args = MagicMock(cluster_id=CID, plugin="p", endpoint="", options=None, credentials_file=None, + credential_ref=None, lun_size=GIB, journal="lun", journal_lun_size=8 * GIB, + journal_options='{"pool": "fast"}', json=True) + with patch.object(array_controller, "register_array", return_value={"id": "arr"}) as reg: + CLIWrapperBase.cluster__set_array(MagicMock(), "set-array", args) + kw = reg.call_args.kwargs + self.assertEqual((kw["journal"], kw["journal_lun_size"], kw["journal_options"]), + ("lun", 8 * GIB, {"pool": "fast"})) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_array_lifecycle.py b/tests/unit/test_array_lifecycle.py new file mode 100644 index 0000000000..24b364cd2e --- /dev/null +++ b/tests/unit/test_array_lifecycle.py @@ -0,0 +1,1299 @@ +"""Multi-attach array LUN mode, phase 4: array plugins, credentials in the +KMS, registration, the LUN lifecycle, and the AWS EBS io2 plugin. + + - KMS generic secrets (LocalKMS, HCPClient): stored plaintext only at the + persistence boundary, read back as SecretStr, own key prefix. + - plugin interface: connect with KMS credentials, ownership tags, default + fence = forced detach; FakeArrayPlugin implements everything in memory. + - array_controller: register / get / delete (credentials never on the + cluster record), node-add LUN provisioning, attachment reconcile on role + changes (never detaches the serving node, never attaches a fenced one), + growth by factor, node release, cluster delete. + - lun_takeover fences by the array's own LUN / host ids. + - aws_ebs against botocore's Stubber. + - v2 API and CLI front-ends. +""" + +import json +import os +import tempfile +import unittest +from contextlib import nullcontext +from typing import ClassVar +from unittest.mock import MagicMock, patch + +from pydantic import SecretStr + +from simplyblock_core import array, multi_attach, utils +from simplyblock_core.array import aws_ebs +from simplyblock_core.array.fake import FakeArrayPlugin +from simplyblock_core.controllers import array_controller, device_controller +from simplyblock_core.kms import KMSException, array_secret_path +from simplyblock_core.kms._fdb import LocalKMS +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode + +PLUGIN = "fake-p4" +CID = "cluster-4" +GIB = 1024 ** 3 + + +# ------------------------------------------------------------------ fakes + +class FakeKV: + def __init__(self): + self.data = {} + + def set(self, k, v): + self.data[k] = v + + def get(self, k): + return self.data.get(k) + + def clear(self, k): + self.data.pop(k, None) + + +class FakeKMS: + """A KMS holding secrets in memory (as plaintext, like the stores), and + wrapping credentials by hex-encoding them (the real envelope format).""" + store: ClassVar[dict] = {} + keks: ClassVar[set] = set() + + def __enter__(self): + return self + + def __exit__(self, *a): + return None + + def ensure_key_encryption_key(self, name): + FakeKMS.keks.add(name) + + def _wrap_bytes(self, kek_name, plaintext): + return f"fake:{kek_name}:" + plaintext.hex() + + def _unwrap_bytes(self, kek_name, wrapped): + prefix = f"fake:{kek_name}:" + if not wrapped.startswith(prefix): + raise KMSException("wrong key") + return bytes.fromhex(wrapped[len(prefix):]) + + from simplyblock_core.kms._base import KMS as _KMS + wrap_secret = _KMS.wrap_secret + unwrap_secret = _KMS.unwrap_secret + del _KMS + + def put_secret(self, path, data): + from simplyblock_core.kms._base import secret_plaintext + FakeKMS.store[path] = secret_plaintext(data) + + def get_secret(self, path): + from simplyblock_core.kms._base import secret_wrapped + if path not in FakeKMS.store: + raise KMSException(f"No secret found at {path}") + return secret_wrapped(FakeKMS.store[path]) + + def delete_secret(self, path): + FakeKMS.store.pop(path, None) + + +class FakeDB: + def __init__(self, cluster, nodes): + self.cluster = cluster + self.nodes = {n.get_id(): n for n in nodes} + + def get_cluster_by_id(self, _cid): + return self.cluster + + def get_storage_nodes_by_cluster_id(self, _cid): + return list(self.nodes.values()) + + def get_storage_node_by_id(self, nid): + return self.nodes[nid] + + def atomic_update(self, obj, fn): + fresh = self.cluster if isinstance(obj, Cluster) else self.nodes[obj.get_id()] + fn(fresh) + return fresh + + +def _cluster(plugin=PLUGIN, multi=True, lun_size=0): + cl = Cluster() + cl.uuid = CID + cl.multi_attach = multi + cl.max_fault_tolerance = 2 + cl.array_plugin = plugin + cl.array_lun_size = lun_size + cl.status = Cluster.STATUS_ACTIVE + # No external KMS in these tests: the local one, explicitly allowed. + cl.allow_local_kms_credentials = True + return cl + + +def _node(nid, sec="", ter="", status=StorageNode.STATUS_ONLINE): + n = StorageNode() + n.uuid = nid + n.cluster_id = CID + n.hostname = f"{nid}_5260" + n.rpc_port = 5260 + n.status = status + n.secondary_node_id, n.tertiary_node_id = sec, ter + n.agent = MagicMock() + n.agent.lun_info.return_value = ([{"by_id_path": f"/dev/disk/by-id/lun-{nid}", + "by_id_paths": [], "device_path": "/dev/sdx"}], None) + n.client = lambda *a, _c=n.agent, **k: _c + return n + + +def _lun_device(owner, lun_id, wwn, size=10 * GIB): + d = NVMeDevice() + d.uuid = f"dev-{owner.get_id()}" + d.node_id = owner.get_id() + d.cluster_id = CID + d.status = NVMeDevice.STATUS_ONLINE + d.multi_attach = True + d.array_lun_id = lun_id + d.lun_wwn = wwn + d.size = size + d.node_paths = {owner.get_id(): {"path": f"/dev/disk/by-id/own-{owner.get_id()}", "bdev": "aio"}} + owner.nvme_devices = [d] + return d + + +class _Case(unittest.TestCase): + def setUp(self): + FakeArrayPlugin.reset() + FakeKMS.store = {} + FakeKMS.keks = set() + array.register_plugin(PLUGIN, FakeArrayPlugin) + self.addCleanup(array.unregister_plugin, PLUGIN) + for target, attr, kw in ( + (array_controller, "_event", {}), + (array_controller, "_sleep", {}), + ("simplyblock_core.kms.create_kms_connection", None, + {"side_effect": lambda c, **k: FakeKMS()}), + ("simplyblock_core.models.lock.DbLock", None, {"side_effect": lambda *a, **k: nullcontext()})): + p = patch(target, **kw) if attr is None else patch.object(target, attr, **kw) + p.start() + self.addCleanup(p.stop) + + def use_db(self, db): + p = patch.object(array_controller, "_db", return_value=db) + p.start() + self.addCleanup(p.stop) + return db + + def plugin(self, cluster): + return array.plugin_for_cluster(cluster) + + +# -------------------------------------------------------------------- KMS + +class TestLocalKmsSecrets(unittest.TestCase): + + def _kms(self): + kv = FakeKV() + with patch("simplyblock_core.db_controller.DBController") as dbc: + dbc.return_value.kv_store = kv + return LocalKMS(MagicMock()), kv + + def test_round_trip_plaintext_only_at_rest(self): + kms, kv = self._kms() + kms.put_secret("cluster/c/array/a", {"key": SecretStr("s3cr3t"), "user": "u"}) + raw = kv.data[b"secrets/cluster/c/array/a"] + self.assertEqual(json.loads(raw), {"key": "s3cr3t", "user": "u"}) + got = kms.get_secret("cluster/c/array/a") + self.assertIsInstance(got["key"], SecretStr) + self.assertEqual(got["key"].get_secret_value(), "s3cr3t") + self.assertNotIn("s3cr3t", repr(got)) + + def test_own_prefix_and_delete(self): + kms, kv = self._kms() + kms.import_data_encryption_keys("p", "kek", (SecretStr("aa"), SecretStr("bb"))) + kms.put_secret("p", {"x": "y"}) + self.assertIn(b"keys/p", kv.data) + self.assertIn(b"secrets/p", kv.data) + kms.delete_secret("p") + self.assertNotIn(b"secrets/p", kv.data) + self.assertIn(b"keys/p", kv.data) + kms.delete_secret("p") # idempotent + with self.assertRaises(KMSException): + kms.get_secret("p") + + def test_refuses_non_string_values(self): + kms, _ = self._kms() + with self.assertRaises(TypeError): + kms.put_secret("p", {"x": {"nested": "secret"}}) + + +class TestHcpSecrets(unittest.TestCase): + + def _client(self): + from simplyblock_core.kms._hcp import HCPClient + c = HCPClient.__new__(HCPClient) + c.kv_mount = "kv" + c.client = MagicMock() + return c + + def test_put_get_delete(self): + import hvac.exceptions + c = self._client() + c.put_secret("cluster/c/array/a", {"k": SecretStr("v")}) + c.client.secrets.kv.v2.create_or_update_secret.assert_called_once_with( + path="cluster/c/array/a", secret={"k": "v"}, mount_point="kv") + c.client.secrets.kv.v2.read_secret_version.return_value = {"data": {"data": {"k": "v"}}} + got = c.get_secret("cluster/c/array/a") + self.assertEqual(got["k"].get_secret_value(), "v") + c.client.secrets.kv.v2.read_secret_version.side_effect = hvac.exceptions.InvalidPath() + with self.assertRaises(KMSException): + c.get_secret("x") + c.client.secrets.kv.v2.delete_metadata_and_all_versions.side_effect = hvac.exceptions.InvalidPath() + c.delete_secret("x") # a missing secret is deleted already + c.client.secrets.kv.v2.delete_metadata_and_all_versions.side_effect = hvac.exceptions.Forbidden() + with self.assertRaises(KMSException): + c.delete_secret("x") + + +# ------------------------------------------------------- plugin interface + +class TestPluginInterface(_Case): + + def test_plugin_for_cluster_connects_with_kms_credentials(self): + cl = _cluster() + cl.array_id, cl.array_has_credentials = "arr-1", True + cl.array_endpoint, cl.array_options = "https://array", {"pool": "p"} + FakeKMS.store[array_secret_path(CID, "arr-1")] = {"password": "pw"} + plugin = array.plugin_for_cluster(cl) + endpoint, creds, opts = FakeArrayPlugin.connected[-1] + self.assertEqual((endpoint, opts), ("https://array", {"pool": "p"})) + self.assertEqual(creds["password"].get_secret_value(), "pw") + self.assertEqual(plugin.cluster_id, CID) + self.assertNotIn("pw", repr(plugin)) + + def test_no_kms_read_without_credentials(self): + cl = _cluster() + with patch("simplyblock_core.kms.create_kms_connection") as kms: + array.plugin_for_cluster(cl) + kms.assert_not_called() + + def test_unreadable_credentials_are_an_array_error(self): + cl = _cluster() + cl.array_id, cl.array_has_credentials = "arr-1", True + with self.assertRaisesRegex(array.ArrayError, "KMS"): + array.plugin_for_cluster(cl) + + def test_connect_failure_is_an_array_error(self): + FakeArrayPlugin.fail.add("connect") + with self.assertRaisesRegex(array.ArrayError, "could not connect"): + array.plugin_for_cluster(_cluster()) + + def test_builtin_aws_ebs_without_entry_point(self): + with patch.object(array, "_entry_point", return_value=None): + self.assertIs(array.plugin_class("aws-ebs"), aws_ebs.AwsEbsPlugin) + + def test_default_fence_is_forced_detach(self): + class Minimal(FakeArrayPlugin): + fence = array.ArrayPlugin.fence + unfence = array.ArrayPlugin.unfence + p = Minimal(cluster_id=CID) + lun = p.create_lun("l", GIB, {}) + p.attach(lun["id"], "h1") + self.assertTrue(p.fence(lun["id"], "h1")) + self.assertIn(("detach", lun["id"], "h1", True), FakeArrayPlugin.calls) + self.assertEqual(p.get(lun["id"])["hosts"], []) + self.assertTrue(p.unfence(lun["id"], "h1")) + self.assertEqual(p.get(lun["id"])["hosts"], ["h1"]) + + def test_fake_lifecycle_and_ownership(self): + p = self.plugin(_cluster()) + lun = p.create_lun("node-a", 5 * GIB, {"x": "1"}) + self.assertEqual(p.create_lun("node-a", 5 * GIB, {})["id"], lun["id"]) # idempotent + self.assertEqual(FakeArrayPlugin.luns[lun["id"]]["tags"][array.TAG_CLUSTER_ID], CID) + p.attach(lun["id"], "h1") + p.attach(lun["id"], "h1") + p.attach(lun["wwn"], "h2") # by WWN + self.assertEqual(p.get(lun["id"])["hosts"], ["h1", "h2"]) + p.resize(lun["id"], 8 * GIB) + with self.assertRaises(array.ArrayError): + p.resize(lun["id"], GIB) + p.detach(lun["id"], "h2") + p.detach(lun["id"], "h2") + self.assertEqual([x["id"] for x in p.list_luns()], [lun["id"]]) + p.delete_lun(lun["id"]) + p.delete_lun(lun["id"]) + self.assertEqual(p.list_luns(), []) + + def test_foreign_lun_is_never_touched(self): + p = self.plugin(_cluster()) + FakeArrayPlugin().add_foreign_lun("lun-x", tags={array.TAG_CLUSTER_ID: "other"}) + for op in (lambda: p.attach("lun-x", "h"), lambda: p.detach("lun-x", "h"), + lambda: p.resize("lun-x", GIB * 9), lambda: p.delete_lun("lun-x"), + lambda: p.get("lun-x")): + with self.assertRaises(array.ForeignLunError): + op() + self.assertIn("lun-x", FakeArrayPlugin.luns) + self.assertEqual(p.list_luns(), []) + + +# ------------------------------------------------------------ registration + +class TestRegistration(_Case): + + def test_register_keeps_only_wrapped_credentials(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + view = array_controller.register_array( + CID, PLUGIN, "https://array", {"az": "z"}, {"password": "pw"}, lun_size=GIB) + cl = db.cluster + self.assertTrue(cl.array_id) + self.assertEqual((cl.array_plugin, cl.array_endpoint, cl.array_options, cl.array_lun_size), + (PLUGIN, "https://array", {"az": "z"}, GIB)) + self.assertTrue(cl.array_has_credentials) + ref = cl.array_credential_ref + self.assertEqual((ref["kind"], ref["backend"], ref["kek_name"]), + ("managed", "local", f"credentials-{CID}")) + self.assertEqual(FakeKMS.store, {}) # nothing at the old KMS path + self.assertNotIn("pw", json.dumps(cl.to_dict(unwrap_secrets=True), default=str)) + self.assertEqual(view["credential_ref"], {"kind": "managed", "backend": "local"}) + # Read back through the reference, bound to array id + endpoint. + self.assertEqual(array.array_credentials(cl)["password"].get_secret_value(), "pw") + self.assertNotIn("credentials", view) + self.assertNotIn("pw", json.dumps(view)) + self.assertEqual(view["capabilities"]["max_attachments"], 16) + self.assertEqual(array_controller.get_arrays(CID)[0]["id"], cl.array_id) + # The connect saw the credentials wrapped. + self.assertIsInstance(FakeArrayPlugin.connected[-1][1]["password"], SecretStr) + + def test_reregister_keeps_id_and_drops_old_secret(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + array_controller.register_array(CID, PLUGIN, credentials={"password": "pw"}) + aid = db.cluster.array_id + array_controller.register_array(CID, PLUGIN) + self.assertEqual(db.cluster.array_id, aid) + self.assertFalse(db.cluster.array_has_credentials) + self.assertEqual(db.cluster.array_credential_ref, {}) + self.assertEqual(FakeKMS.store, {}) + + def test_reregister_removes_a_legacy_kms_secret(self): + cl = _cluster() + cl.array_id, cl.array_has_credentials = "arr-1", True + FakeKMS.store[array_secret_path(CID, "arr-1")] = {"password": "old"} + db = self.use_db(FakeDB(cl, [])) + array_controller.register_array(CID, PLUGIN, credentials={"password": "new"}) + self.assertEqual(FakeKMS.store, {}) + self.assertEqual(db.cluster.array_credential_ref["kind"], "managed") + self.assertEqual(array.array_credentials(db.cluster)["password"].get_secret_value(), "new") + + def test_local_kms_needs_the_opt_in(self): + cl = _cluster(plugin="") + cl.allow_local_kms_credentials = False + db = self.use_db(FakeDB(cl, [])) + with self.assertRaisesRegex(ValueError, "allow_local_kms_credentials"): + array_controller.register_array(CID, PLUGIN, credentials={"password": "pw"}) + self.assertEqual((db.cluster.array_plugin, db.cluster.array_credential_ref), ("", {})) + # Without credentials (an instance role) nothing is stored: allowed. + array_controller.register_array(CID, PLUGIN) + self.assertEqual(db.cluster.array_plugin, PLUGIN) + + def test_endpoint_rewrite_does_not_release_the_credentials(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + array_controller.register_array(CID, PLUGIN, "https://array.example", + credentials={"password": "pw"}) + db.cluster.array_endpoint = "https://attacker.example" + with self.assertRaisesRegex(array.ArrayError, "different context"): + array.array_credentials(db.cluster) + + def test_external_vault_reference(self): + from simplyblock_core.kms import HCPClient + vault = MagicMock(spec=HCPClient) + vault.__enter__.return_value = vault + vault.read_kv_secret.return_value = {"password": SecretStr("ext")} + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + ref = {"kind": "vault-kv", "mount": "secret", "path": "arrays/a1"} + with patch("simplyblock_core.kms.create_kms_connection", side_effect=lambda c, **k: vault): + view = array_controller.register_array(CID, PLUGIN, credential_ref=ref) + self.assertEqual(FakeArrayPlugin.connected[-1][1]["password"].get_secret_value(), "ext") + self.assertEqual(array.array_credentials(db.cluster)["password"].get_secret_value(), "ext") + self.assertEqual(db.cluster.array_credential_ref["path"], "arrays/a1") + self.assertEqual(view["credential_ref"]["kind"], "vault-kv") + vault.read_kv_secret.assert_called_with("secret", "arrays/a1", None) + + def test_reference_refusals(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + with self.assertRaisesRegex(ValueError, "not both"): + array_controller.register_array(CID, PLUGIN, credentials={"p": "x"}, + credential_ref={"kind": "vault-kv", "mount": "m", "path": "p"}) + with self.assertRaisesRegex(ValueError, "external secret"): + array_controller.register_array(CID, PLUGIN, credential_ref={ + "kind": "managed", "backend": "local", "kek_name": "k", "wrapped": "w"}) + with self.assertRaisesRegex(ValueError, "not a valid reference"): + array_controller.register_array(CID, PLUGIN, credential_ref={"kind": "nope"}) + # A Vault reference on a cluster whose KMS is not Vault. + with self.assertRaisesRegex(ValueError, "could not be read"): + array_controller.register_array(CID, PLUGIN, credential_ref={ + "kind": "vault-kv", "mount": "m", "path": "p"}) + self.assertEqual(db.cluster.array_plugin, "") + + def test_refusals_save_nothing(self): + db = self.use_db(FakeDB(_cluster(plugin="", multi=False), [])) + with self.assertRaisesRegex(ValueError, "multi-attach"): + array_controller.register_array(CID, PLUGIN) + db.cluster.multi_attach = True + FakeArrayPlugin.fail.add("connect") + with self.assertRaisesRegex(ValueError, "connect"): + array_controller.register_array(CID, PLUGIN, credentials={"p": "x"}) + FakeArrayPlugin.fail.clear() + FakeArrayPlugin.max_attachments = 2 # FTT2 needs 3 hosts per LUN + with self.assertRaisesRegex(ValueError, "at most 2"): + array_controller.register_array(CID, PLUGIN) + with self.assertRaisesRegex(ValueError, "no array plugin"): + array_controller.register_array(CID, "missing") + self.assertEqual((db.cluster.array_plugin, db.cluster.array_id, FakeKMS.store), ("", "", {})) + + def test_delete(self): + a = _node("a") + db = self.use_db(FakeDB(_cluster(plugin=""), [a])) + array_controller.register_array(CID, PLUGIN, credentials={"p": "x"}) + aid = db.cluster.array_id + with self.assertRaises(KeyError): + array_controller.delete_array(CID, "other") + _lun_device(a, "lun-1", "w") + with self.assertRaisesRegex(ValueError, "LUN"): + array_controller.delete_array(CID, aid) + array_controller.delete_array(CID, aid, force=True) + self.assertEqual((db.cluster.array_plugin, db.cluster.array_id), ("", "")) + self.assertEqual(FakeKMS.store, {}) + self.assertEqual(array_controller.get_arrays(CID), []) + + +# ------------------------------------------------------------- node add + +class TestProvisioning(_Case): + + def _api(self, appear_after=1): + api = MagicMock() + calls = {"n": 0} + + def inv(): + calls["n"] += 1 + disks = [{"type": "disk", "name": "nvme0n1", "serial": "local", "size": GIB}] + if calls["n"] > appear_after: + lun = next(iter(FakeArrayPlugin.luns.values())) + disks.append({"type": "disk", "name": "nvme3n1", "serial": lun["wwn"], + "wwn": "eui.x", "size": lun["size"], "by_id_path": "/dev/disk/by-id/x", + "numa_node": 0}) + return disks, None + api.get_blockdevices.side_effect = inv + return api + + def test_provision_creates_attaches_and_waits(self): + cl = _cluster(lun_size=20 * GIB) + info = {"hostname": "worker-1", "system_id": "sys", + "cloud_instance": {"cloud": "amazon", "id": "i-0abc"}} + entry, host = array_controller.provision_node_lun( + cl, self._api(), info, {"socket": 0}, [{"serial": "local", "name": "nvme0n1"}]) + lun = next(iter(FakeArrayPlugin.luns.values())) + self.assertEqual(host, "host-worker-1") + self.assertEqual(FakeArrayPlugin.hosts[host]["initiators"]["ec2_instance_id"], "i-0abc") + self.assertEqual(lun["name"], "simplyblock-worker-1-s0") + self.assertEqual(lun["size"], 20 * GIB) + self.assertEqual(lun["hosts"], [host]) + self.assertEqual((entry["serial"], entry["lun_wwn"], entry["array_lun_id"], entry["name"]), + (lun["wwn"], lun["wwn"], lun["id"], "nvme3n1")) + # A retried add reuses the LUN. + entry2, _ = array_controller.provision_node_lun( + cl, self._api(appear_after=0), info, {"socket": 0}, [entry]) + self.assertEqual(entry2["array_lun_id"], lun["id"]) + self.assertEqual(len(FakeArrayPlugin.luns), 1) + + def test_san_initiators_reach_the_array_and_the_host_scans(self): + cl = _cluster(lun_size=GIB) + api = self._api(appear_after=2) + api.scan_new_luns.return_value = ({"scanned": ["/sys/class/scsi_host/host0/scan"]}, None) + info = {"hostname": "worker-2", "system_id": "sys", + "san_initiators": {"iscsi_iqn": "iqn.2026-01.io.sb:w2", "fc_wwpns": ["2100"], + "unknown": "dropped"}} + _, host = array_controller.provision_node_lun(cl, api, info, {"socket": 1}, []) + inits = FakeArrayPlugin.hosts[host]["initiators"] + self.assertEqual((inits["iscsi_iqn"], inits["fc_wwpns"]), ("iqn.2026-01.io.sb:w2", ["2100"])) + self.assertNotIn("unknown", inits) + # One scan per inventory poll until the LUN showed up. + self.assertEqual(api.scan_new_luns.call_count, api.get_blockdevices.call_count) + self.assertGreaterEqual(api.scan_new_luns.call_count, 3) + + def test_agent_without_scan_endpoint_still_provisions(self): + api = self._api() + api.scan_new_luns.side_effect = RuntimeError("404") + entry, _ = array_controller.provision_node_lun( + _cluster(lun_size=GIB), api, {"hostname": "w"}, {"socket": 0}, []) + self.assertTrue(entry["array_lun_id"]) + + def test_existing_node_initiators_come_from_its_agent(self): + n = _node("a") + n.agent.info.return_value = ({"san_initiators": {"nvme_hostnqn": "nqn.h:a"}}, None) + plugin = self.plugin(_cluster()) + db = FakeDB(_cluster(), [n]) + host = array_controller.ensure_node_host(plugin, n, db) + self.assertEqual(FakeArrayPlugin.hosts[host]["initiators"]["nvme_hostnqn"], "nqn.h:a") + self.assertEqual(n.array_host_id, host) + + def test_a_lun_behind_a_multipath_map_is_the_map(self): + cl = _cluster(lun_size=GIB) + api = MagicMock() + + def inv(): + lun = next(iter(FakeArrayPlugin.luns.values()), None) + if lun is None: + return [], None + path = {"type": "disk", "serial": lun["wwn"], "wwn": "", "size": GIB, + "multipath_map": "mpatha", "holders": ["dm-3"]} + return [dict(path, name="sdb", by_id_path="/dev/disk/by-id/wwn-sdb"), + dict(path, name="sdc", by_id_path="/dev/disk/by-id/wwn-sdc"), + {"type": "mpath", "name": "mpatha", "kname": "dm-3", "serial": lun["wwn"], + "wwn": "", "size": GIB, "paths": ["sdb", "sdc"], "numa_node": 0, + "by_id_path": "/dev/disk/by-id/dm-uuid-mpath-3" + lun["wwn"]}], None + api.get_blockdevices.side_effect = inv + api.scan_new_luns.return_value = ({}, None) + entry, _ = array_controller.provision_node_lun(cl, api, {"hostname": "w"}, {"socket": 0}, []) + self.assertEqual((entry["name"], entry["kname"]), ("mpatha", "dm-3")) + self.assertTrue(entry["by_id"].startswith("/dev/disk/by-id/dm-uuid-mpath-")) + + def test_lun_that_never_appears_fails(self): + cl = _cluster(lun_size=GIB) + with self.assertRaisesRegex(array_controller.ArrayLifecycleError, "did not appear"): + array_controller.provision_node_lun(cl, self._api(appear_after=10 ** 6), + {"hostname": "h"}, {}, [], timeout=0) + + def test_array_failure_fails(self): + FakeArrayPlugin.fail.add("create_lun") + with self.assertRaisesRegex(array_controller.ArrayLifecycleError, "array"): + array_controller.provision_node_lun(_cluster(lun_size=GIB), self._api(), + {"hostname": "h"}, {}, []) + + def test_config_entry_and_journal_flag(self): + entry = {"serial": "vol0a", "array_lun_id": "vol-0a", "lun_wwn": "vol0a"} + out = array_controller.add_lun_to_config([{"serial": "local"}], entry) + self.assertEqual(out, [{"serial": "local", "journal": True}, entry]) + out = array_controller.add_lun_to_config(out, entry) # re-add: replaced, not doubled + self.assertEqual(len(out), 2) + two = array_controller.add_lun_to_config([{"serial": "a"}, {"serial": "b"}], entry) + self.assertFalse(any(e.get("journal") for e in two)) + + def test_resolve_and_mark_carry_the_array_identity(self): + inv = [{"name": "nvme3n1", "device_path": "/dev/nvme3n1", "serial": "vol0a", "wwn": "eui.1", + "size": GIB, "type": "disk"}] + resolved, missing = utils.resolve_lblk_entries( + [{"serial": "vol0a", "lun_wwn": "vol0a", "array_lun_id": "vol-0a"}], inv) + self.assertEqual(missing, []) + self.assertEqual((resolved[0]["lun_wwn"], resolved[0]["array_lun_id"]), ("vol0a", "vol-0a")) + node = _node("a") + dev = NVMeDevice() + dev.serial_number = "vol0a" + multi_attach.mark_lun_devices(_cluster(), node, [dev], resolved) + self.assertEqual((dev.lun_wwn, dev.array_lun_id), ("vol0a", "vol-0a")) + self.assertEqual(multi_attach.array_lun_ref(dev), "vol-0a") + + def test_gates(self): + self.assertFalse(array_controller.provisions_luns(_cluster(lun_size=0))) + self.assertFalse(array_controller.provisions_luns(_cluster(plugin="", lun_size=GIB))) + self.assertFalse(array_controller.provisions_luns(_cluster(multi=False, lun_size=GIB))) + self.assertTrue(array_controller.provisions_luns(_cluster(lun_size=GIB))) + + +# ------------------------------------------------------------ reconcile + +class TestReconcile(_Case): + + def _bed(self): + a = _node("a", "b", "c") + b, c, d = _node("b"), _node("c"), _node("d") + db = self.use_db(FakeDB(_cluster(), [a, b, c, d])) + p = self.plugin(db.cluster) + lun = p.create_lun("a", 10 * GIB, {}) + a.array_host_id = p.ensure_host("a", {}) + p.attach(lun["id"], a.array_host_id) + dev = _lun_device(a, lun["id"], lun["wwn"]) + return db, p, a, dev + + def test_attaches_roles_and_records_paths(self): + db, p, a, dev = self._bed() + rep = array_controller.reconcile_lun_attachments(CID) + self.assertEqual(rep["errors"], {}) + self.assertEqual(sorted(p.get(dev.array_lun_id)["hosts"]), ["host-a", "host-b", "host-c"]) + self.assertEqual(sorted(dev.array_attached_node_ids), ["a", "b", "c"]) + self.assertEqual(dev.node_paths["b"]["path"], "/dev/disk/by-id/lun-b") + self.assertIn("a", dev.node_paths) + self.assertEqual(db.nodes["b"].array_host_id, "host-b") + # Steady state: no array call at all. + FakeArrayPlugin.calls.clear() + FakeArrayPlugin.connected.clear() + array_controller.reconcile_lun_attachments(CID) + self.assertEqual((FakeArrayPlugin.calls, FakeArrayPlugin.connected), ([], [])) + + def test_role_move_attaches_first_then_detaches(self): + db, p, a, dev = self._bed() + array_controller.reconcile_lun_attachments(CID) + a.tertiary_node_id = "d" + FakeArrayPlugin.calls.clear() + array_controller.reconcile_lun_attachments(CID) + ops = [c[:3] for c in FakeArrayPlugin.calls if c[0] in ("attach", "detach")] + self.assertEqual(ops, [("attach", dev.array_lun_id, "host-d"), + ("detach", dev.array_lun_id, "host-c")]) + self.assertNotIn("c", dev.node_paths) + self.assertIn("d", dev.node_paths) + + def test_serving_node_is_never_detached_fenced_never_attached(self): + db, p, a, dev = self._bed() + array_controller.reconcile_lun_attachments(CID) + # A takeover moved the LUN to c and fenced b. + a.status = StorageNode.STATUS_OFFLINE + dev.serving_node_id = "c" + dev.fenced_node_ids = ["b"] + p.fence(dev.array_lun_id, "host-b") + a.secondary_node_id, a.tertiary_node_id = "d", "" + array_controller.reconcile_lun_attachments(CID) + hosts = p.get(dev.array_lun_id)["hosts"] + self.assertIn("host-c", hosts) # serving, lost its role: kept + self.assertNotIn("host-b", hosts) # fenced: not re-attached + self.assertIn("host-d", hosts) + self.assertIn("c", dev.node_paths) + + def test_not_visible_yet_is_retried(self): + db, p, a, dev = self._bed() + db.nodes["b"].agent.lun_info.return_value = ([], None) + rep = array_controller.reconcile_lun_attachments(CID, path_timeout=0) + self.assertIn(dev.get_id(), rep["errors"]) + self.assertNotIn("b", dev.node_paths) + db.nodes["b"].agent.lun_info.return_value = ([{"by_id_path": "/dev/disk/by-id/z"}], None) + array_controller.reconcile_lun_attachments(CID) + self.assertEqual(dev.node_paths["b"]["path"], "/dev/disk/by-id/z") + + def test_noop_without_array_and_for_operator_luns(self): + db, p, a, dev = self._bed() + db.cluster.array_plugin = "" + FakeArrayPlugin.calls.clear() + array_controller.reconcile_lun_attachments(CID) + db.cluster.array_plugin = PLUGIN + dev.array_lun_id = "" # presented by the operator: not ours + array_controller.reconcile_lun_attachments(CID) + self.assertEqual(FakeArrayPlugin.calls, []) + + def test_failure_of_one_lun_is_reported(self): + db, p, a, dev = self._bed() + FakeArrayPlugin.fail.add("attach") + rep = array_controller.reconcile_after_role_change(CID) + self.assertIn(dev.get_id(), rep["errors"]) + array_controller._event.assert_called() + + +# --------------------------------------------------------------- growth + +class TestGrowth(_Case): + + def test_grow_by_factor_and_wait_visible(self): + a = _node("a", "b") + b = _node("b") + db = self.use_db(FakeDB(_cluster(), [a, b])) + p = self.plugin(db.cluster) + lun = p.create_lun("a", 10 * GIB, {}) + dev = _lun_device(a, lun["id"], lun["wwn"]) + dev.node_paths["b"] = {"path": "/dev/disk/by-id/lun-b", "bdev": ""} + a.agent.rescan_block_device.return_value = ({"size": 15 * GIB}, None) + b.agent.rescan_block_device.side_effect = [({"size": 10 * GIB}, None), ({"size": 15 * GIB}, None)] + out = array_controller.grow_luns(CID, 1.5) + self.assertEqual(out["errors"], {}) + self.assertEqual(out["luns"][dev.get_id()]["new_size"], 15 * GIB) + self.assertEqual(FakeArrayPlugin.luns[lun["id"]]["size"], 15 * GIB) + self.assertEqual(b.agent.rescan_block_device.call_count, 2) + + def test_grow_to_a_size_is_idempotent(self): + a, b = _node("a"), _node("b") + db = self.use_db(FakeDB(_cluster(lun_size=10 * GIB), [a, b])) + p = self.plugin(db.cluster) + la, lb = p.create_lun("a", 10 * GIB, {}), p.create_lun("b", 12 * GIB, {}) + da, db_ = _lun_device(a, la["id"], la["wwn"]), _lun_device(b, lb["id"], lb["wwn"]) + a.agent.rescan_block_device.return_value = ({"size": 15 * GIB}, None) + b.agent.rescan_block_device.return_value = ({"size": 15 * GIB}, None) + out = array_controller.grow_luns(CID, size=15 * GIB) + self.assertEqual(out["errors"], {}) + self.assertEqual((FakeArrayPlugin.luns[la["id"]]["size"], FakeArrayPlugin.luns[lb["id"]]["size"]), + (15 * GIB, 15 * GIB)) + self.assertEqual(out["luns"][db_.get_id()]["old_size"], 12 * GIB) + # Nodes added later get LUNs of the new size. + self.assertEqual(db.cluster.array_lun_size, 15 * GIB) + # Repeated (after a failure elsewhere): nothing is resized again. + FakeArrayPlugin.calls.clear() + out = array_controller.grow_luns(CID, size=15 * GIB) + self.assertEqual(out["errors"], {}) + self.assertFalse([c for c in FakeArrayPlugin.calls if c[0] == "resize"]) + self.assertEqual(FakeArrayPlugin.luns[la["id"]]["size"], 15 * GIB) + self.assertEqual(da.get_id() in out["luns"], True) + + def test_a_failed_growth_to_a_size_keeps_the_lun_size(self): + a = _node("a") + db = self.use_db(FakeDB(_cluster(lun_size=10 * GIB), [a])) + p = self.plugin(db.cluster) + lun = p.create_lun("a", 10 * GIB, {}) + _lun_device(a, lun["id"], lun["wwn"]) + a.agent.rescan_block_device.return_value = ({"size": 10 * GIB}, None) + out = array_controller.grow_luns(CID, size=15 * GIB, visible_timeout=0) + self.assertTrue(out["errors"]) + self.assertEqual(db.cluster.array_lun_size, 10 * GIB) + + def test_factor_or_size(self): + self.use_db(FakeDB(_cluster(), [_node("a")])) + for kw in ({}, {"factor": 2, "size": GIB}, {"size": 0}, {"size": "x"}): + with self.subTest(kw=kw), self.assertRaises(ValueError): + array_controller.grow_luns(CID, **kw) + + def test_grow_devices_passes_the_size(self): + self._prechecked() + with patch.object(array_controller, "grow_luns", return_value={"luns": {}, "errors": {}}) as grow_luns, patch.object(device_controller, "_grow_cluster_devices", + return_value={"grown": False, "factor": 1.0, "devices": [], "errors": {}, + "reason": "", "shrunk": []}): + out = device_controller.grow_cluster_devices(CID, lun_size=6 << 40) + grow_luns.assert_called_once_with(CID, None, size=6 << 40) + self.assertEqual(out["errors"], {}) + + def test_refusals_and_errors(self): + a = _node("a") + db = self.use_db(FakeDB(_cluster(), [a])) + for bad in (1, 0.5, "x", float("inf")): + with self.assertRaises(ValueError): + array_controller.grow_luns(CID, bad) + p = self.plugin(db.cluster) + lun = p.create_lun("a", GIB, {}) + dev = _lun_device(a, lun["id"], lun["wwn"]) + a.agent.rescan_block_device.return_value = ({"size": GIB}, None) + out = array_controller.grow_luns(CID, 2, visible_timeout=0) + self.assertIn("not visible", out["errors"][dev.get_id()]) + FakeArrayPlugin.fail.add("resize") + out = array_controller.grow_luns(CID, 2) + self.assertIn("array resize", out["errors"][dev.get_id()]) + db.cluster.array_plugin = "" + with self.assertRaisesRegex(ValueError, "array"): + array_controller.grow_luns(CID, 2) + + def _prechecked(self, errors=None): + """Patch the pre-array check of grow_cluster_devices.""" + db = MagicMock() + db.get_cluster_by_id.return_value = MagicMock(device_mode=device_controller.constants.DEVICE_MODE_LBLK) + for target, attr, kw in ((device_controller, "DBController", {"return_value": db}), + (device_controller, "_growth_devices", {"return_value": []}), + (device_controller, "_resizable_errors", {"return_value": errors or {}})): + p = patch.object(target, attr, **kw) + p.start() + self.addCleanup(p.stop) + + def test_a_device_that_cannot_grow_leaves_the_array_alone(self): + # Checked before the array is touched: LUNs grown at the array with + # the cluster refusing would be grown again by a re-run. + self._prechecked({"d1": "Storage node must be online, current status: offline"}) + with patch.object(array_controller, "grow_luns") as grow_luns, patch.object(device_controller, "_grow_cluster_devices") as inner: + out = device_controller.grow_cluster_devices(CID, lun_factor=2) + grow_luns.assert_not_called() + inner.assert_not_called() + self.assertFalse(out["grown"]) + self.assertIn("no LUN was grown", out["reason"]) + self.assertEqual(out["luns"], {}) + + def test_a_failed_lun_grows_nothing(self): + self._prechecked() + # Growth is uniform (PR #1572): a LUN that did not grow at the array + # would cap the factor at 1, so nothing on the cluster is grown. + with patch.object(array_controller, "grow_luns", + return_value={"luns": {}, "errors": {"d1": "array resize failed"}}), patch.object(device_controller, "_grow_cluster_devices") as inner: + out = device_controller.grow_cluster_devices(CID, lun_factor=2) + inner.assert_not_called() + self.assertFalse(out["grown"]) + self.assertEqual(out["errors"], {"d1": "array resize failed"}) + self.assertEqual(out["luns"], {}) + + def test_grown_luns_then_uniform_growth(self): + self._prechecked() + luns = {"d1": {"lun_id": "vol-1", "old_size": 1, "new_size": 2}} + with patch.object(array_controller, "grow_luns", return_value={"luns": luns, "errors": {}}), patch.object(device_controller, "_grow_cluster_devices", + return_value={"grown": True, "factor": 2.0, "devices": [], "errors": {}, + "reason": "", "shrunk": []}) as inner: + out = device_controller.grow_cluster_devices(CID, lun_factor=2) + inner.assert_called_once_with(CID) + self.assertTrue(out["grown"]) + self.assertEqual(out["luns"], luns) + + def test_served_elsewhere_is_not_resized_on_the_owner(self): + dev = NVMeDevice() + dev.multi_attach, dev.node_id, dev.serving_node_id = True, "a", "b" + dev.status, dev.bdev_type, dev.alceml_bdev, dev.nvme_bdev = NVMeDevice.STATUS_ONLINE, "aio", "al", "nv" + snode = StorageNode() + snode.uuid, snode.status = "a", StorageNode.STATUS_ONLINE + cluster = MagicMock(device_mode=device_controller.constants.DEVICE_MODE_LBLK) + with patch.object(device_controller.tasks_controller, "get_active_dev_restart_task", return_value=None): + with self.assertRaisesRegex(ValueError, "served by node b"): + device_controller._check_resizable(cluster, dev, snode) + + +# --------------------------------------------------------- release / delete + +class TestReleaseAndDelete(_Case): + + def test_release_node_detaches_without_deleting(self): + a, b = _node("a", "b"), _node("b") + db = self.use_db(FakeDB(_cluster(), [a, b])) + p = self.plugin(db.cluster) + la = p.create_lun("a", GIB, {}) + lb = p.create_lun("b", GIB, {}) + a.array_host_id, b.array_host_id = "host-a", "host-b" + for lun in (la, lb): + p.attach(lun["id"], "host-a") + p.attach(lun["id"], "host-b") + _lun_device(a, la["id"], la["wwn"]) + _lun_device(b, lb["id"], lb["wwn"]) + array_controller.release_node_luns(a) + self.assertEqual(p.get(la["id"])["hosts"], []) + self.assertEqual(p.get(lb["id"])["hosts"], ["host-b"]) + self.assertEqual(len(FakeArrayPlugin.luns), 2) + + def test_release_never_detaches_a_lun_the_node_serves(self): + a, b = _node("a"), _node("b") + db = self.use_db(FakeDB(_cluster(), [a, b])) + p = self.plugin(db.cluster) + lb = p.create_lun("b", GIB, {}) + a.array_host_id = "host-a" + p.attach(lb["id"], "host-a") + dev = _lun_device(b, lb["id"], lb["wwn"]) + dev.serving_node_id = "a" + array_controller.release_node_luns(a) + self.assertEqual(p.get(lb["id"])["hosts"], ["host-a"]) + + def test_delete_cluster_deletes_only_its_luns_and_secret(self): + cl = _cluster() + self.use_db(FakeDB(cl, [])) + cl.array_id, cl.array_has_credentials = "arr", True + FakeKMS.store[array_secret_path(CID, "arr")] = {"k": "v"} + p = self.plugin(cl) + lun = p.create_lun("a", GIB, {}) + p.attach(lun["id"], "h") + FakeArrayPlugin().add_foreign_lun("lun-foreign", tags={array.TAG_CLUSTER_ID: "other"}) + array_controller.delete_cluster_luns(cl) + self.assertEqual(sorted(FakeArrayPlugin.luns), ["lun-foreign"]) + self.assertEqual(FakeKMS.store, {}) + + def test_delete_cluster_uses_recorded_luns_when_listing_is_unsupported(self): + cl = _cluster() + a = _node("a") + self.use_db(FakeDB(cl, [a])) + p = self.plugin(cl) + lun = p.create_lun("a", GIB, {}) + _lun_device(a, lun["id"], lun["wwn"]) + with patch.object(FakeArrayPlugin, "list_luns", side_effect=NotImplementedError): + array_controller.delete_cluster_luns(cl) + self.assertEqual(FakeArrayPlugin.luns, {}) + + def test_delete_cluster_failure(self): + cl = _cluster() + self.use_db(FakeDB(cl, [])) + p = self.plugin(cl) + p.create_lun("a", GIB, {}) + FakeArrayPlugin.fail.add("delete_lun") + with self.assertRaisesRegex(ValueError, "could not be deleted"): + array_controller.delete_cluster_luns(cl) + array_controller.delete_cluster_luns(cl, force=True) # logged, goes on + + def test_noop_on_regular_cluster(self): + array_controller.delete_cluster_luns(_cluster(multi=False)) + array_controller.release_node_luns(_node("a")) + self.assertEqual(FakeArrayPlugin.calls, []) + + +# ------------------------------------------------------------ takeover + +class TestTakeoverFenceIds(unittest.TestCase): + + def test_fence_uses_array_ids(self): + from simplyblock_core.controllers import lun_takeover + FakeArrayPlugin.reset() + array.register_plugin(PLUGIN, FakeArrayPlugin) + self.addCleanup(array.unregister_plugin, PLUGIN) + failed = _node("a") + failed.array_host_id = "i-0abc" + dev = NVMeDevice() + dev.lun_wwn, dev.array_lun_id = "vol0abc", "vol-0abc" + lun_takeover._fence(_cluster(), dev, failed) + self.assertEqual(FakeArrayPlugin.calls, [("fence", "vol-0abc", "i-0abc")]) + failed.array_host_id, dev.array_lun_id = "", "" + lun_takeover._fence(_cluster(), dev, failed) + self.assertEqual(FakeArrayPlugin.calls[-1], ("fence", "vol0abc", "a")) + + +# ------------------------------------------------------------ AWS EBS + +class TestAwsEbs(unittest.TestCase): + + def setUp(self): + import boto3 + from botocore.stub import Stubber + self.ec2 = boto3.session.Session( + aws_access_key_id="x", aws_secret_access_key="y", region_name="us-east-1").client("ec2") + self.stub = Stubber(self.ec2) + self.stub.activate() + self.addCleanup(self.stub.deactivate) + self.p = aws_ebs.AwsEbsPlugin(options={"az": "us-east-1a", "poll_interval": 0, "timeout": 5}, + cluster_id=CID) + self.p.ec2 = self.ec2 + self.p._sleep = lambda s: None + + def _vol(self, vid="vol-0abc", state="available", size=10, atts=(), tags=None): + if tags is None: + tags = {array.TAG_CLUSTER_ID: CID} + return {"VolumeId": vid, "State": state, "Size": size, "AvailabilityZone": "us-east-1a", + "Attachments": [{"InstanceId": i, "State": s, "VolumeId": vid} for i, s in atts], + "Tags": [{"Key": k, "Value": v} for k, v in tags.items()]} + + def _describe(self, vol, vid="vol-0abc"): + self.stub.add_response("describe_volumes", {"Volumes": [vol]}, {"VolumeIds": [vid]}) + + def test_ids(self): + self.assertEqual(aws_ebs.volume_wwn("vol-0abc"), "vol0abc") + self.assertEqual(aws_ebs.volume_id_of("vol0abc"), "vol-0abc") + self.assertEqual(aws_ebs.volume_id_of("vol-0abc"), "vol-0abc") + with self.assertRaises(array.ArrayError): + aws_ebs.volume_id_of("0x600a") + + def test_create_new_volume(self): + self.stub.add_response("describe_volumes", {"Volumes": []}) + self.stub.add_response("create_volume", {"VolumeId": "vol-0abc", "State": "creating"}, { + "AvailabilityZone": "us-east-1a", "Size": 20, "VolumeType": "io2", "Iops": 3000, + "MultiAttachEnabled": True, "Encrypted": True, + "TagSpecifications": [{"ResourceType": "volume", "Tags": [ + {"Key": "Name", "Value": "lun-a"}, + {"Key": "role", "Value": "data"}, + {"Key": array.TAG_CLUSTER_ID, "Value": CID}, + {"Key": array.TAG_LUN_NAME, "Value": "lun-a"}]}]}) + self._describe(self._vol(state="creating", size=20)) + self._describe(self._vol(size=20)) + out = self.p.create_lun("lun-a", int(19.5 * GIB), {"role": "data"}) + self.assertEqual(out, {"id": "vol-0abc", "wwn": "vol0abc", "size": 20 * GIB}) + self.stub.assert_no_pending_responses() + + def test_create_finds_existing(self): + self.stub.add_response("describe_volumes", {"Volumes": [self._vol()]}) + self._describe(self._vol()) + self.assertEqual(self.p.create_lun("lun-a", GIB, {})["id"], "vol-0abc") + self.stub.assert_no_pending_responses() + + def test_attach_picks_a_free_device_and_waits(self): + self._describe(self._vol(atts=[("i-other", "attached")])) + self.stub.add_response("describe_instances", {"Reservations": [{"Instances": [{ + "InstanceId": "i-1", "RootDeviceName": "/dev/xvda", + "BlockDeviceMappings": [{"DeviceName": "/dev/sdf"}]}]}]}, {"InstanceIds": ["i-1"]}) + self.stub.add_response("attach_volume", {}, {"VolumeId": "vol-0abc", "InstanceId": "i-1", + "Device": "/dev/sdg"}) + self._describe(self._vol(atts=[("i-other", "attached"), ("i-1", "attaching")])) + self._describe(self._vol(atts=[("i-other", "attached"), ("i-1", "attached")])) + self.p.attach("vol0abc", "i-1") + self.stub.assert_no_pending_responses() + + def test_attach_already_attached_is_a_noop(self): + self._describe(self._vol(atts=[("i-1", "attached")])) + self._describe(self._vol(atts=[("i-1", "attached")])) + self.p.attach("vol-0abc", "i-1") + self.stub.assert_no_pending_responses() + + def test_fence_is_forced_detach_and_waits_until_gone(self): + self._describe(self._vol(atts=[("i-1", "attached"), ("i-2", "attached")])) + self.stub.add_response("detach_volume", {}, {"VolumeId": "vol-0abc", "InstanceId": "i-1", + "Force": True}) + self._describe(self._vol(atts=[("i-1", "detaching"), ("i-2", "attached")])) + self._describe(self._vol(atts=[("i-2", "attached")])) + self.assertTrue(self.p.fence("vol-0abc", "i-1")) + self.stub.assert_no_pending_responses() + + def test_fence_polls_fast(self): + slept = [] + self.p._sleep = slept.append + self._describe(self._vol(atts=[("i-1", "attached")])) + self.stub.add_response("detach_volume", {}, {"VolumeId": "vol-0abc", "InstanceId": "i-1", + "Force": True}) + self._describe(self._vol(atts=[("i-1", "detaching")])) + self._describe(self._vol(atts=[])) + self.assertTrue(self.p.fence("vol-0abc", "i-1")) + self.assertEqual(slept, [0.2]) + self.p.options["fence_poll_interval"] = 0.05 + self.assertEqual(self.p._fence_poll, 0.05) + + def test_detach_not_attached_is_a_noop(self): + self._describe(self._vol(atts=[("i-2", "attached")])) + self.p.detach("vol-0abc", "i-1") + self.stub.assert_no_pending_responses() + + def test_detach_timeout(self): + self.p.options["timeout"] = 0 + self._describe(self._vol(atts=[("i-1", "attached")])) + self.stub.add_response("detach_volume", {}, {"VolumeId": "vol-0abc", "InstanceId": "i-1", + "Force": False}) + self._describe(self._vol(atts=[("i-1", "busy")])) + with self.assertRaisesRegex(array.ArrayError, "timed out"): + self.p.detach("vol-0abc", "i-1") + + def test_resize_grows_and_waits_for_optimizing(self): + self._describe(self._vol(size=10)) + self.stub.add_response("modify_volume", {}, {"VolumeId": "vol-0abc", "Size": 15}) + self.stub.add_response("describe_volumes_modifications", {"VolumesModifications": [ + {"VolumeId": "vol-0abc", "ModificationState": "modifying"}]}, {"VolumeIds": ["vol-0abc"]}) + self.stub.add_response("describe_volumes_modifications", {"VolumesModifications": [ + {"VolumeId": "vol-0abc", "ModificationState": "optimizing"}]}, {"VolumeIds": ["vol-0abc"]}) + self.p.resize("vol-0abc", 15 * GIB) + self.stub.assert_no_pending_responses() + + def test_resize_never_shrinks(self): + self._describe(self._vol(size=10)) + with self.assertRaisesRegex(array.ArrayError, "shrinking"): + self.p.resize("vol-0abc", 5 * GIB) + + def test_resize_failed_modification(self): + self._describe(self._vol(size=10)) + self.stub.add_response("modify_volume", {}, {"VolumeId": "vol-0abc", "Size": 11}) + self.stub.add_response("describe_volumes_modifications", {"VolumesModifications": [ + {"VolumeId": "vol-0abc", "ModificationState": "failed", "StatusMessage": "nope"}]}) + with self.assertRaisesRegex(array.ArrayError, "failed"): + self.p.resize("vol-0abc", 11 * GIB) + + def test_foreign_volume_is_refused(self): + for tags in ({}, {array.TAG_CLUSTER_ID: "other"}): + self._describe(self._vol(atts=[("i-1", "attached")], tags=tags)) + with self.assertRaises(array.ForeignLunError): + self.p.detach("vol-0abc", "i-1", force=True) + self._describe(self._vol(tags=tags)) + with self.assertRaises(array.ForeignLunError): + self.p.delete_lun("vol-0abc") + self.stub.assert_no_pending_responses() + + def test_get_and_list(self): + self._describe(self._vol(size=8, atts=[("i-1", "attached"), ("i-2", "detached")])) + self.assertEqual(self.p.get("vol-0abc"), + {"id": "vol-0abc", "wwn": "vol0abc", "size": 8 * GIB, "hosts": ["i-1"], + "state": "available"}) + self.stub.add_response("describe_volumes", {"Volumes": [self._vol(), self._vol("vol-1", "deleting")]}, + {"Filters": [{"Name": f"tag:{array.TAG_CLUSTER_ID}", "Values": [CID]}]}) + self.assertEqual([x["id"] for x in self.p.list_luns()], ["vol-0abc"]) + + def test_delete_detaches_then_deletes(self): + self._describe(self._vol(state="in-use", atts=[("i-1", "attached")])) + self._describe(self._vol(state="in-use", atts=[("i-1", "attached")])) + self.stub.add_response("detach_volume", {}, {"VolumeId": "vol-0abc", "InstanceId": "i-1", + "Force": True}) + self._describe(self._vol()) + self._describe(self._vol()) + self.stub.add_response("delete_volume", {}, {"VolumeId": "vol-0abc"}) + self.p.delete_lun("vol-0abc") + self.stub.assert_no_pending_responses() + + def test_delete_missing_is_a_noop(self): + self.stub.add_client_error("describe_volumes", "InvalidVolume.NotFound") + self.p.delete_lun("vol-0abc") + + def test_ensure_host_checks_zone(self): + self.stub.add_response("describe_instances", {"Reservations": [{"Instances": [{ + "InstanceId": "i-1", "Placement": {"AvailabilityZone": "us-east-1a"}}]}]}) + self.assertEqual(self.p.ensure_host("w1", {"ec2_instance_id": "i-1"}), "i-1") + self.stub.add_response("describe_instances", {"Reservations": [{"Instances": [{ + "InstanceId": "i-2", "Placement": {"AvailabilityZone": "us-east-1b"}}]}]}) + with self.assertRaisesRegex(array.ArrayError, "cannot cross zones"): + self.p.ensure_host("w2", {"ec2_instance_id": "i-2"}) + with self.assertRaisesRegex(array.ArrayError, "no EC2 instance id"): + self.p.ensure_host("w3", {"hostname": "w3"}) + + def test_ensure_host_by_private_ip(self): + # Docker deployments start the node agent with WITHOUT_CLOUD_INFO, so + # the instance is found by the node's private IP within the zone. + expected = {"Filters": [ + {"Name": "private-ip-address", "Values": ["172.31.36.102"]}, + {"Name": "availability-zone", "Values": ["us-east-1a"]}, + {"Name": "instance-state-name", "Values": ["pending", "running"]}]} + self.stub.add_response("describe_instances", {"Reservations": [{"Instances": [{ + "InstanceId": "i-7"}]}]}, expected) + self.stub.add_response("describe_instances", {"Reservations": [{"Instances": [{ + "InstanceId": "i-7", "Placement": {"AvailabilityZone": "us-east-1a"}}]}]}) + self.assertEqual(self.p.ensure_host("w1", {"hostname": "w1", "ip": "172.31.36.102"}), "i-7") + self.stub.assert_no_pending_responses() + + def test_ensure_host_by_ip_refuses_none_or_several(self): + self.stub.add_response("describe_instances", {"Reservations": []}) + with self.assertRaisesRegex(array.ArrayError, "0 running instances"): + self.p.ensure_host("w1", {"ip": "10.0.0.9"}) + self.stub.add_response("describe_instances", {"Reservations": [ + {"Instances": [{"InstanceId": "i-1"}, {"InstanceId": "i-2"}]}]}) + with self.assertRaisesRegex(array.ArrayError, "2 running instances"): + self.p.ensure_host("w1", {"ip": "10.0.0.9"}) + + def test_host_initiators_carry_the_ip(self): + out = array_controller.host_initiators("w1", {}, "", "172.31.36.102") + self.assertEqual(out, {"hostname": "w1", "ip": "172.31.36.102"}) + + def test_connect(self): + p = aws_ebs.AwsEbsPlugin(cluster_id=CID) + with self.assertRaisesRegex(array.ArrayError, "'az'"): + p.connect("", {}, {}) + with self.assertRaisesRegex(array.ArrayError, "go together"): + p.connect("", {"aws_access_key_id": SecretStr("k")}, {"az": "us-east-1a"}) + with self.assertRaisesRegex(array.ArrayError, "volume_type"): + p.connect("", {}, {"az": "us-east-1a", "volume_type": "gp3"}) + with patch("boto3.session.Session") as sess: + client = sess.return_value.client.return_value + client.describe_availability_zones.return_value = {"AvailabilityZones": [{"ZoneName": "us-east-1a"}]} + p.connect("", {"aws_access_key_id": SecretStr("k"), "aws_secret_access_key": SecretStr("s")}, + {"az": "us-east-1a"}) + sess.assert_called_once_with(aws_access_key_id="k", aws_secret_access_key="s", + aws_session_token=None, region_name="us-east-1") + sess.return_value.client.assert_called_once_with("ec2", endpoint_url=None) + self.assertEqual(p.capabilities()["max_attachments"], 16) + self.assertNotIn("s", repr(p).replace("AwsEbsPlugin", "").replace("cluster", "")) + + def test_instance_role_without_credentials(self): + p = aws_ebs.AwsEbsPlugin(cluster_id=CID) + with patch("boto3.session.Session") as sess: + sess.return_value.client.return_value.describe_availability_zones.return_value = { + "AvailabilityZones": [{}]} + p.connect("https://ec2.vpce", {}, {"az": "eu-west-1b"}) + sess.assert_called_once_with(aws_access_key_id=None, aws_secret_access_key=None, + aws_session_token=None, region_name="eu-west-1") + sess.return_value.client.assert_called_once_with("ec2", endpoint_url="https://ec2.vpce") + + +# ----------------------------------------------------------- front-ends + +class TestApi(unittest.TestCase): + + def _client(self): + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from simplyblock_web.api.v2._dependencies import _lookup_cluster + from simplyblock_web.api.v2.cluster import array as array_api + app = FastAPI() + app.include_router(array_api.api, prefix="/clusters/{cluster_id}/arrays") + app.dependency_overrides[_lookup_cluster] = lambda: _cluster() + return TestClient(app) + + def test_post_get_delete(self): + view = {"id": "arr", "plugin": "aws-ebs", "endpoint": "", "options": {"az": "z"}, + "lun_size": GIB, "has_credentials": True, "capabilities": {"max_attachments": 16}} + cid = "00000000-0000-0000-0000-000000000001" + with patch.object(array_controller, "register_array", return_value=view) as reg, \ + patch.object(array_controller, "get_arrays", return_value=[dict(view, capabilities=None)]), \ + patch.object(array_controller, "delete_array") as dele: + c = self._client() + r = c.post(f"/clusters/{cid}/arrays/", json={ + "plugin": "aws-ebs", "options": {"az": "z"}, "lun_size": "1G", + "credentials": {"aws_secret_access_key": "TOPSECRET"}}) + self.assertEqual(r.status_code, 201, r.text) + self.assertNotIn("TOPSECRET", r.text) + args = reg.call_args.args + self.assertIsInstance(args[4]["aws_secret_access_key"], SecretStr) + self.assertEqual(args[5], utils.parse_size("1G")) + r = c.get(f"/clusters/{cid}/arrays/") + self.assertEqual(r.json()[0]["id"], "arr") + self.assertNotIn("credentials", r.json()[0]) + self.assertEqual(c.get(f"/clusters/{cid}/arrays/arr").status_code, 200) + self.assertEqual(c.get(f"/clusters/{cid}/arrays/nope").status_code, 404) + self.assertEqual(c.delete(f"/clusters/{cid}/arrays/arr?force=true").status_code, 204) + dele.assert_called_once_with(_cluster().get_id(), "arr", force=True) + dele.side_effect = ValueError("LUNs exist") + self.assertEqual(c.delete(f"/clusters/{cid}/arrays/arr").status_code, 409) + reg.side_effect = ValueError("bad plugin") + r = c.post(f"/clusters/{cid}/arrays/", json={"plugin": "x"}) + # A refusal is 400 with the reason as a string; 422 is the + # request-validation answer generated clients parse as a list. + self.assertEqual((r.status_code, r.json()["detail"]), (400, "bad plugin")) + + +class TestCli(unittest.TestCase): + + def test_set_array_reads_and_wraps_credentials(self): + from simplyblock_cli.clibase import CLIWrapperBase + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f: + json.dump({"aws_access_key_id": "AK", "aws_secret_access_key": "SK"}, f) + self.addCleanup(os.unlink, f.name) + args = MagicMock(cluster_id=CID, plugin="aws-ebs", endpoint="", options='{"az": "z"}', + credentials_file=f.name, credential_ref=None, lun_size=GIB, journal="local", + journal_lun_size=0, journal_options=None, json=True) + with patch.object(array_controller, "register_array", return_value={"id": "arr"}) as reg: + CLIWrapperBase.cluster__set_array(MagicMock(), "set-array", args) + a = reg.call_args.args + self.assertEqual((a[0], a[1], a[3], a[5]), (CID, "aws-ebs", {"az": "z"}, GIB)) + self.assertEqual(a[4]["aws_secret_access_key"].get_secret_value(), "SK") + self.assertNotIn("SK", repr(a[4])) + + def test_bad_credentials_file_does_not_echo_it(self): + from simplyblock_cli.clibase import CLIWrapperBase + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f: + f.write("SECRET-NOT-JSON") + self.addCleanup(os.unlink, f.name) + args = MagicMock(options=None, credentials_file=f.name) + with self.assertRaises(ValueError) as cm: + CLIWrapperBase.cluster__set_array(MagicMock(), "set-array", args) + self.assertNotIn("SECRET", str(cm.exception)) + + +# ------------------------------------------------------------- agent + +class TestAgent(unittest.TestCase): + + def test_amazon_info_reports_the_zone(self): + from simplyblock_web.api.internal import _node_info + doc = {"instanceId": "i-1", "instanceType": "m5", "privateIp": "10.0.0.1", + "availabilityZone": "us-east-1a", "region": "us-east-1"} + token = MagicMock(text="tok") + ident = MagicMock() + ident.json.return_value = doc + with patch.object(_node_info.requests, "put", return_value=token) as put, patch.object(_node_info.requests, "get", return_value=ident) as get: + info = _node_info._amazon_info() + self.assertEqual((info["id"], info["availability_zone"], info["region"]), + ("i-1", "us-east-1a", "us-east-1")) + self.assertIn("/latest/api/token", put.call_args.args[0]) + self.assertEqual(get.call_args.kwargs["headers"], {"X-aws-ec2-metadata-token": "tok"}) + + def test_amazon_info_none_off_aws(self): + from simplyblock_web.api.internal import _node_info + with patch.object(_node_info.requests, "put", side_effect=OSError("no route")): + self.assertIsNone(_node_info._amazon_info()) + + def test_rescan_dm_multipath_map(self): + from simplyblock_web import node_utils + root = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, root) + + def mk(rel, content=""): + path = os.path.join(root, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as fh: + fh.write(content) + mk("sys/class/block/dm-2/size", "4096") + mk("sys/class/block/dm-2/dm/name", "mpatha") + mk("sys/class/block/dm-2/dm/uuid", "mpath-3600a0") + mk("sys/class/block/dm-2/slaves/sdb/x") + mk("sys/class/block/dm-2/slaves/sdc/x") + mk("sys/class/block/sdb/device/rescan") + mk("sys/class/block/sdc/device/rescan") + + def _map(p): + return os.path.join(root, p.lstrip("/")) if p.startswith("/sys/") else p + real_exists, real_isdir, real_listdir = os.path.exists, os.path.isdir, os.listdir + writes = [] + with patch.object(node_utils.os.path, "realpath", return_value="/dev/dm-2"), \ + patch.object(node_utils.os.path, "exists", side_effect=lambda p: real_exists(_map(p))), \ + patch.object(node_utils.os.path, "isdir", side_effect=lambda p: real_isdir(_map(p))), \ + patch.object(node_utils.os, "listdir", side_effect=lambda p: real_listdir(_map(p))), \ + patch.object(node_utils, "_read_sysfs", + side_effect=lambda p: open(_map(p)).read().strip() if real_exists(_map(p)) else ""), \ + patch.object(node_utils, "_write_sysfs", side_effect=lambda p, v: writes.append(p)), \ + patch.object(node_utils.shell_utils, "run_command", return_value=("ok", "", 0)) as run: + res = node_utils.rescan_block_device("/dev/mapper/mpatha") + self.assertEqual(writes, ["/sys/class/block/sdb/device/rescan", "/sys/class/block/sdc/device/rescan"]) + run.assert_called_once_with("multipathd resize map mpatha") + self.assertEqual(res["size"], 4096 * 512) + self.assertIn("multipathd:mpatha", res["rescanned"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_cluster_floors.py b/tests/unit/test_cluster_floors.py new file mode 100644 index 0000000000..e747a3442b --- /dev/null +++ b/tests/unit/test_cluster_floors.py @@ -0,0 +1,389 @@ +"""Unit tests for the erasure-coding resource floors (simplyblock_core.floors) +and the places that enforce them. + + - cluster_floors: the five floors for 2+1, 1+1, 2+2 and 4+2, with and + without raid_devices; the single-node and two-node topology overrides. + - count_data_devices: journal devices never count (floor 5). + - floor_violations: every violated floor is named. + - evaluate: online resources, the node left out of a planned operation. + - Activation: fresh activation refuses below a floor; the failure-domain + minimum is max(k+2, n+k). + - Monitor: DEGRADED (never SUSPENDED) while online resources are below a + floor, named in the cluster event. + - Shutdown preconditions: refused below a floor, force overrides. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import cluster_ops, floors, storage_node_ops +from simplyblock_core.controllers.cluster_expansion import planner +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.nvme_device import JMDevice, NVMeDevice +from simplyblock_core.models.storage_node import StorageNode + + +def _cluster(ndcs, npcs, raid_devices=False, is_single_node=False, two_node_arbitration=False, + enable_failure_domain=False, ha_type="ha"): + cl = Cluster() + cl.uuid = "cluster-1" + cl.distr_ndcs = ndcs + cl.distr_npcs = npcs + cl.raid_devices = raid_devices + cl.is_single_node = is_single_node + cl.two_node_arbitration = two_node_arbitration + cl.enable_failure_domain = enable_failure_domain + cl.ha_type = ha_type + cl.max_fault_tolerance = max(npcs, 1) + cl.status = Cluster.STATUS_ACTIVE + return cl + + +def _dev(status=NVMeDevice.STATUS_ONLINE, uuid="dev"): + d = NVMeDevice() + d.uuid = uuid + d.status = status + return d + + +def _node(uuid, devices=1, status=StorageNode.STATUS_ONLINE, failure_domain=-1, + device_statuses=None, with_jm=False): + n = StorageNode() + n.uuid = uuid + n.cluster_id = "cluster-1" + n.status = status + n.mgmt_ip = f"10.0.0.{abs(hash(uuid)) % 250 + 1}" + n.failure_domain = failure_domain + statuses = device_statuses if device_statuses is not None else [NVMeDevice.STATUS_ONLINE] * devices + n.nvme_devices = [_dev(s, f"{uuid}-dev-{i}") for i, s in enumerate(statuses)] + if with_jm: + jm = JMDevice() + jm.uuid = f"{uuid}-jm" + jm.status = NVMeDevice.STATUS_ONLINE + n.jm_device = jm + return n + + +# --------------------------------------------------------------------------- +# cluster_floors +# --------------------------------------------------------------------------- + +class TestClusterFloors(unittest.TestCase): + + def _floors(self, ndcs, npcs, members=8, **kw): + cl = _cluster(ndcs, npcs, **kw) + return floors.cluster_floors(cl, uses_failure_domains=kw.get("enable_failure_domain", False), + member_nodes=members) + + def test_2_1(self): + f = self._floors(2, 1) + self.assertEqual((f.nodes, f.devices_per_node, f.data_devices, f.failure_domains), (4, 1, 4, None)) + f = self._floors(2, 1, raid_devices=True) + self.assertEqual((f.nodes, f.data_devices), (4, 3)) + self.assertIn("raid_devices", f.data_devices_rule) + + def test_1_1(self): + f = self._floors(1, 1) + self.assertEqual((f.nodes, f.data_devices), (3, 3)) + self.assertEqual(self._floors(1, 1, raid_devices=True).data_devices, 2) + + def test_2_2(self): + f = self._floors(2, 2) + self.assertEqual((f.nodes, f.data_devices), (6, 6)) + self.assertEqual(self._floors(2, 2, raid_devices=True).data_devices, 4) + + def test_4_2(self): + f = self._floors(4, 2) + self.assertEqual((f.nodes, f.data_devices), (8, 8)) + self.assertEqual(self._floors(4, 2, raid_devices=True).data_devices, 6) + + def test_failure_domain_floor_is_n_plus_k_only_when_used(self): + self.assertEqual(self._floors(4, 2, enable_failure_domain=True).failure_domains, 6) + self.assertEqual(self._floors(2, 1, enable_failure_domain=True).failure_domains, 3) + self.assertIsNone(self._floors(2, 1).failure_domains) + + def test_single_node_override(self): + for kw in ({"is_single_node": True, "members": 3}, {"members": 1}): + f = self._floors(2, 1, **kw) + self.assertEqual((f.nodes, f.data_devices), (1, 3), kw) + self.assertEqual(f.nodes_rule, "single-node cluster") + cl = _cluster(1, 0) + cl.activated_single_node = True + f = floors.cluster_floors(cl, False, member_nodes=1) + self.assertEqual((f.nodes, f.data_devices), (1, 1)) + + def test_two_node_override(self): + for kw in ({"two_node_arbitration": True, "members": 2}, {"members": 2}): + f = self._floors(1, 1, **kw) + # Two nodes, but the data-device floor stays the cluster rule. + self.assertEqual((f.nodes, f.data_devices), (2, 3), kw) + self.assertEqual(f.nodes_rule, "two-node cluster") + self.assertEqual(self._floors(1, 1, raid_devices=True, members=2).data_devices, 2) + + def test_volume_scheme_override(self): + cl = _cluster(4, 2) + f = floors.cluster_floors(cl, False, member_nodes=8, ndcs=2, npcs=1) + self.assertEqual((f.nodes, f.data_devices), (4, 4)) + + def test_mock_flags_never_read_truthy(self): + """A MagicMock stand-in for the cluster (spec'd or not) must not turn a + child mock into raid_devices / two_node / single-node.""" + cl = MagicMock(spec=Cluster) + cl.distr_ndcs, cl.distr_npcs = 2, 1 + f = floors.cluster_floors(cl, False, member_nodes=5) + self.assertEqual((f.nodes, f.data_devices), (4, 4)) + + +# --------------------------------------------------------------------------- +# count_data_devices (floor 5: journal devices never count) +# --------------------------------------------------------------------------- + +class TestCountDataDevices(unittest.TestCase): + + def test_journal_is_not_a_data_device(self): + # A dedicated journal device lives in jm_device, outside nvme_devices; + # a device still flagged STATUS_JM inside the list is skipped as well. + n1 = _node("n1", device_statuses=[NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_JM], with_jm=True) + n2 = _node("n2", devices=2, with_jm=True) + total, per_node = floors.count_data_devices([n1, n2]) + self.assertEqual(total, 3) + self.assertEqual(per_node, {"n1": 1, "n2": 2}) + + def test_only_serving_statuses_count(self): + n = _node("n1", device_statuses=[ + NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE, + NVMeDevice.STATUS_UNAVAILABLE, NVMeDevice.STATUS_NEW, NVMeDevice.STATUS_FAILED, + NVMeDevice.STATUS_REMOVED, NVMeDevice.STATUS_FAILED_AND_MIGRATED]) + total, per_node = floors.count_data_devices([n]) + self.assertEqual((total, per_node["n1"]), (3, 3)) + + def test_a_node_without_devices_is_listed_with_zero(self): + total, per_node = floors.count_data_devices([_node("n1", devices=0), _node("n2")]) + self.assertEqual((total, per_node), (1, {"n1": 0, "n2": 1})) + + def test_secondary_only_nodes_are_skipped(self): + sec = _node("sec", devices=0) + sec.is_secondary_node = True + total, per_node = floors.count_data_devices([sec, _node("n1")]) + self.assertEqual((total, per_node), (1, {"n1": 1})) + + +# --------------------------------------------------------------------------- +# floor_violations / evaluate +# --------------------------------------------------------------------------- + +class TestViolations(unittest.TestCase): + + def test_every_violated_floor_is_named(self): + f = floors.Floors(nodes=4, devices_per_node=1, data_devices=4, failure_domains=3) + v = floors.floor_violations(f, online_nodes=3, data_devices=2, per_node={"a": 2, "b": 0}, online_fds=2) + self.assertEqual(v, [ + "nodes online 3 < floor 4 (n+2k)", + "node b has 0 online data devices < floor 1 (data devices per node)", + "data devices online 2 < floor 4 (n+2k)", + "failure domains online 2 < floor 3 (n+k)", + ]) + + def test_no_violation_is_empty(self): + f = floors.Floors(nodes=4, devices_per_node=1, data_devices=4, failure_domains=None) + self.assertEqual(floors.floor_violations(f, 4, 4, {"a": 1, "b": 1, "c": 1, "d": 1}, None), []) + + def test_evaluate_judges_online_resources(self): + cl = _cluster(2, 1) + nodes = [_node(f"n{i}") for i in range(4)] + [_node("n4", status=StorageNode.STATUS_OFFLINE)] + self.assertEqual(floors.evaluate(cl, nodes), []) + nodes[3].status = StorageNode.STATUS_OFFLINE + self.assertEqual(floors.evaluate(cl, nodes), [ + "nodes online 3 < floor 4 (n+2k)", + "data devices online 3 < floor 4 (n+2k)", + ]) + + def test_evaluate_without_a_node_keeps_the_membership(self): + """Shutting one node of three down does not turn a 1+1 cluster into a + two-node cluster: the excluded node still counts as a member.""" + cl = _cluster(1, 1) + nodes = [_node("n0", devices=2), _node("n1", devices=2), _node("n2", devices=2)] + self.assertEqual(floors.evaluate(cl, nodes), []) + self.assertEqual(floors.evaluate(cl, nodes, exclude_node_id="n2"), + ["nodes online 2 < floor 3 (n+2k)"]) + + def test_evaluate_failure_domains(self): + cl = _cluster(2, 1, enable_failure_domain=True) + nodes = [_node(f"n{i}", failure_domain=i % 3) for i in range(6)] + self.assertEqual(floors.evaluate(cl, nodes), []) + for n in nodes: + n.failure_domain = n.failure_domain % 2 + self.assertEqual(floors.evaluate(cl, nodes), ["failure domains online 2 < floor 3 (n+k)"]) + + def test_evaluate_volume_override(self): + cl = _cluster(1, 1) + nodes = [_node(f"n{i}") for i in range(3)] + self.assertEqual(floors.evaluate(cl, nodes), []) + self.assertEqual(floors.evaluate(cl, nodes, ndcs=2, npcs=1), + ["nodes online 3 < floor 4 (n+2k)", "data devices online 3 < floor 4 (n+2k)"]) + + def test_migration_affinity_floor(self): + self.assertEqual(floors.migration_affinity_floor(_cluster(2, 1)), "device") + self.assertEqual(floors.migration_affinity_floor(_cluster(2, 1, raid_devices=True)), "relaxed") + + +# --------------------------------------------------------------------------- +# Activation +# --------------------------------------------------------------------------- + +class TestActivation(unittest.TestCase): + + def test_fresh_activation_floor_check(self): + cl = _cluster(2, 1) + nodes = [_node(f"n{i}") for i in range(4)] + self.assertEqual(cluster_ops.activation_floor_violations(cl, nodes, nodes), []) + nodes.append(_node("n4", devices=0)) + self.assertEqual(cluster_ops.activation_floor_violations(cl, nodes, nodes), + ["node n4 has 0 online data devices < floor 1 (data devices per node)"]) + + def test_single_node_activation_floor(self): + cl = _cluster(1, 0, is_single_node=True) + nodes = [_node("n0")] + self.assertEqual(cluster_ops.activation_floor_violations(cl, nodes, nodes), []) + cl = _cluster(2, 1, is_single_node=True) + nodes = [_node("n0", devices=3)] + self.assertEqual(cluster_ops.activation_floor_violations(cl, nodes, nodes), []) + nodes = [_node("n0", devices=2)] + self.assertEqual(cluster_ops.activation_floor_violations(cl, nodes, nodes), + ["data devices online 2 < floor 3 (n+k, single-node cluster)"]) + + def test_reactivation_gate_is_the_old_device_count(self): + self.assertEqual(cluster_ops.activation_minimum_devices(_cluster(2, 1), False), 4) + self.assertEqual(cluster_ops.activation_minimum_devices(_cluster(2, 1), True), 3) + + def test_fd_activation_minimum_is_max_of_spare_rule_and_floor(self): + self.assertEqual(planner.fd_activation_min_domains(1, 2), 3) # 2+1: k+2 == n+k + self.assertEqual(planner.fd_activation_min_domains(2, 4), 6) # 4+2: n+k binds + self.assertEqual(planner.fd_activation_min_domains(1, 1), 3) # 1+1: k+2 binds + self.assertIsNone(planner.fd_activation_domain_count_violation(2, 6, ndcs=4)) + msg = planner.fd_activation_domain_count_violation(2, 5, ndcs=4) + self.assertIn("at least 6 distinct failure domains (n+k", msg) + # Without ndcs the former k+2 rule is unchanged. + self.assertIsNone(planner.fd_activation_domain_count_violation(1, 3)) + self.assertIsNotNone(planner.fd_activation_domain_count_violation(1, 2)) + + +# --------------------------------------------------------------------------- +# Monitor +# --------------------------------------------------------------------------- + +class TestMonitorFloors(unittest.TestCase): + + def _status(self, cluster, nodes): + from simplyblock_core.services import storage_node_monitor as mod + mod._floor_violations_reported.clear() + events = MagicMock() + with patch.object(mod, "db") as mock_db, patch.object(mod, "cluster_events", events), \ + patch.object(mod, "is_new_migrated_node", return_value=False): + mock_db.get_cluster_by_id.return_value = cluster + mock_db.get_primary_storage_nodes_by_cluster_id.return_value = nodes + mock_db.get_job_tasks.return_value = [] + return mod.get_next_cluster_status("cluster-1"), events + + def test_healthy_cluster_at_the_floor_is_active(self): + status, events = self._status(_cluster(2, 1), [_node(f"n{i}") for i in range(4)]) + self.assertEqual(status, Cluster.STATUS_ACTIVE) + events.cluster_floor_violation.assert_not_called() + + def test_below_the_node_floor_is_degraded_and_named(self): + status, events = self._status(_cluster(2, 1), [_node(f"n{i}", devices=2) for i in range(3)]) + self.assertEqual(status, Cluster.STATUS_DEGRADED) + _, violations = events.cluster_floor_violation.call_args.args + self.assertEqual(violations, ["nodes online 3 < floor 4 (n+2k)"]) + + def test_below_the_device_floor_is_degraded_not_suspended(self): + # Every node online, the data devices below n+2k: the old rules say + # ACTIVE (n+k devices are there), the floors say DEGRADED. + nodes = [_node(f"n{i}") for i in range(4)] + nodes[3].nvme_devices = [] + status, _ = self._status(_cluster(2, 1), nodes) + self.assertEqual(status, Cluster.STATUS_DEGRADED) + + def test_raid_devices_lowers_the_device_floor_for_the_monitor(self): + nodes = [_node(f"n{i}") for i in range(4)] + nodes[3].nvme_devices = [_dev(NVMeDevice.STATUS_ONLINE)] + nodes[2].nvme_devices = [] + cl = _cluster(2, 1, raid_devices=True) + status, events = self._status(cl, nodes) + # The per-node floor still degrades; the cluster-wide one no longer does. + self.assertEqual(status, Cluster.STATUS_DEGRADED) + _, violations = events.cluster_floor_violation.call_args.args + self.assertEqual(violations, ["node n2 has 0 online data devices < floor 1 (data devices per node)"]) + + def test_suspension_rules_are_untouched(self): + # k=1 with two affected nodes: SUSPENDED by the existing rules, and + # the floors never turn that into DEGRADED. + nodes = [_node(f"n{i}") for i in range(5)] + nodes[0].status = StorageNode.STATUS_OFFLINE + nodes[1].status = StorageNode.STATUS_OFFLINE + status, _ = self._status(_cluster(2, 1), nodes) + self.assertEqual(status, Cluster.STATUS_SUSPENDED) + + def test_event_once_per_change_and_restored(self): + from simplyblock_core.services import storage_node_monitor as mod + cl = _cluster(2, 1) + short = [_node(f"n{i}", devices=2) for i in range(3)] + whole = short + [_node("n3", devices=2)] + mod._floor_violations_reported.clear() + events = MagicMock() + with patch.object(mod, "db") as mock_db, patch.object(mod, "cluster_events", events), \ + patch.object(mod, "is_new_migrated_node", return_value=False): + mock_db.get_cluster_by_id.return_value = cl + mock_db.get_job_tasks.return_value = [] + mock_db.get_primary_storage_nodes_by_cluster_id.return_value = short + mod.get_next_cluster_status("cluster-1") + mod.get_next_cluster_status("cluster-1") + self.assertEqual(events.cluster_floor_violation.call_count, 1) + mock_db.get_primary_storage_nodes_by_cluster_id.return_value = whole + self.assertEqual(mod.get_next_cluster_status("cluster-1"), Cluster.STATUS_ACTIVE) + events.cluster_floor_restored.assert_called_once() + + +# --------------------------------------------------------------------------- +# Shutdown preconditions +# --------------------------------------------------------------------------- + +class TestShutdownFloors(unittest.TestCase): + + def _db(self, cluster, nodes): + db = MagicMock() + db.get_cluster_by_id.return_value = cluster + db.get_storage_nodes_by_cluster_id.return_value = nodes + db.get_storage_node_by_id.side_effect = lambda nid: next(n for n in nodes if n.get_id() == nid) + return db + + def test_shutdown_below_a_floor_is_refused(self): + nodes = [_node(f"n{i}", devices=2) for i in range(4)] + with patch.object(storage_node_ops, "DBController", return_value=self._db(_cluster(2, 1), nodes)): + allowed, reason = storage_node_ops.check_node_shutdown_preconditions("n0") + self.assertFalse(allowed) + self.assertIn("nodes online 3 < floor 4 (n+2k)", reason) + + def test_force_overrides_the_floor(self): + nodes = [_node(f"n{i}", devices=2) for i in range(4)] + tc = MagicMock() + tc.get_active_node_tasks.return_value = [] + tc.get_active_node_restart_task.return_value = [] + with patch.object(storage_node_ops, "DBController", return_value=self._db(_cluster(2, 1), nodes)), \ + patch.object(storage_node_ops, "tasks_controller", tc): + allowed, reason = storage_node_ops.check_node_shutdown_preconditions("n0", force=True) + self.assertTrue(allowed, reason) + + def test_shutdown_with_headroom_passes_the_floor(self): + nodes = [_node(f"n{i}", devices=2) for i in range(5)] + tc = MagicMock() + tc.get_active_node_tasks.return_value = [] + tc.get_active_node_restart_task.return_value = [] + with patch.object(storage_node_ops, "DBController", return_value=self._db(_cluster(2, 1), nodes)), \ + patch.object(storage_node_ops, "tasks_controller", tc): + allowed, reason = storage_node_ops.check_node_shutdown_preconditions("n0") + self.assertTrue(allowed, reason) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_credentials.py b/tests/unit/test_credentials.py new file mode 100644 index 0000000000..2319232688 --- /dev/null +++ b/tests/unit/test_credentials.py @@ -0,0 +1,520 @@ +"""Credentials for external systems: KMS secret envelope + CredentialProvider.""" +import base64 +import json +from unittest.mock import MagicMock + +import hvac.exceptions +import pytest +from kmip.core import enums +from kmip.pie import objects as kmip_objects +from pydantic import SecretStr + +from simplyblock_core.credentials import ( + CONTEXT_FIELD, + Credential, + CredentialError, + CredentialProvider, + KMIPSecretCredentialRef, + ManagedCredentialRef, + VaultKVCredentialRef, + kek_name_for, +) +from simplyblock_core.kms import KMS, HCPClient, KMIPClient, KMSException, LocalKMS +from simplyblock_core.models.cluster import Cluster + +CTX = "array/a1/https://pure01.example:443" + + +class FakeKMS(KMS): + """Reversible, keyed 'encryption' so tests can see wrong-key failures.""" + + def __init__(self): + self.keks: set[str] = set() + self.created: list[str] = [] + + def create_data_encryption_keys(self, path, kek_name): ... + def import_data_encryption_keys(self, path, kek_name, keys): ... + def get_data_encryption_keys(self, path, kek_name): ... + def delete_data_encryption_keys(self, path): ... + + def create_key_encryption_key(self, name): + self.created.append(name) + self.keks.add(name) + + def delete_key_encryption_key(self, name): + self.keks.discard(name) + + def ensure_key_encryption_key(self, name): + if name not in self.keks: + self.create_key_encryption_key(name) + + def put_secret(self, path, data): ... + def get_secret(self, path): ... + def delete_secret(self, path): ... + + def _wrap_bytes(self, kek_name, plaintext): + if kek_name not in self.keks: + raise KMSException("no such KEK") + return base64.b64encode(kek_name.encode() + b"|" + plaintext).decode() + + def _unwrap_bytes(self, kek_name, wrapped): + if kek_name not in self.keks: + raise KMSException("no such KEK") + prefix, _, plaintext = base64.b64decode(wrapped).partition(b"|") + if prefix != kek_name.encode(): + raise KMSException("wrong KEK") + return plaintext + + +def _fields(**kw): + return {k: SecretStr(v) for k, v in kw.items()} + + +def _plain(fields): + return {k: v.get_secret_value() for k, v in fields.items()} + + +# -- Envelope (KMS.wrap_secret / unwrap_secret) ----------------------------- + +class TestEnvelope: + + def setup_method(self): + self.kms = FakeKMS() + self.kms.create_key_encryption_key("k") + + def test_round_trip(self): + wrapped = self.kms.wrap_secret("k", _fields(username="u", password="p"), CTX) + assert _plain(self.kms.unwrap_secret("k", wrapped, CTX)) == { + "username": "u", "password": "p"} + + def test_other_context_is_refused(self): + wrapped = self.kms.wrap_secret("k", _fields(password="p"), CTX) + with pytest.raises(KMSException, match="different context"): + self.kms.unwrap_secret("k", wrapped, "array/a1/https://evil.example:443") + + def test_tampered_payload_is_refused(self): + wrapped = self.kms.wrap_secret("k", _fields(password="p"), CTX) + raw = bytearray(base64.b64decode(wrapped)) + raw[-3] ^= 0x01 # inside the JSON payload, after the digest + with pytest.raises(KMSException, match="integrity"): + self.kms.unwrap_secret("k", base64.b64encode(bytes(raw)).decode(), CTX) + + def test_truncated_blob_is_refused(self): + wrapped = base64.b64encode(b"k|short").decode() + with pytest.raises(KMSException, match="integrity"): + self.kms.unwrap_secret("k", wrapped, CTX) + + def test_unknown_version_is_refused(self): + import hashlib + payload = json.dumps({"v": 99, "ctx": CTX, "fields": {"a": "b"}}).encode() + wrapped = self.kms._wrap_bytes("k", hashlib.sha256(payload).digest() + payload) + with pytest.raises(KMSException, match="unsupported format"): + self.kms.unwrap_secret("k", wrapped, CTX) + + def test_empty_secret_is_refused(self): + with pytest.raises(KMSException): + self.kms.wrap_secret("k", {}, CTX) + + def test_backend_without_wrap_support_says_so(self): + class NoWrap(FakeKMS): + _wrap_bytes = KMS._wrap_bytes + with pytest.raises(NotImplementedError, match="cannot wrap"): + NoWrap().wrap_secret("k", _fields(a="b"), CTX) + + +# -- Backends --------------------------------------------------------------- + +class TestLocalKMSWrapping: + + def _kms(self): + return object.__new__(LocalKMS) # no FDB needed for wrap/unwrap + + def test_round_trip(self): + kms = self._kms() + wrapped = kms.wrap_secret("ignored", _fields(token="t"), CTX) + assert wrapped.startswith("local:") + assert _plain(kms.unwrap_secret("ignored", wrapped, CTX)) == {"token": "t"} + + def test_foreign_ciphertext_is_refused(self): + with pytest.raises(KMSException, match="local KMS"): + self._kms().unwrap_secret("k", "vault:v1:abc", CTX) + + def test_corrupt_encoding_is_refused(self): + with pytest.raises(KMSException): + self._kms().unwrap_secret("k", "local:!!!", CTX) + + +class _FakeKVStore: + def __init__(self): + self.data = {} + + def get(self, key): + return self.data.get(key) + + def set(self, key, value): + self.data[key] = value + + def clear(self, key): + self.data.pop(key, None) + + +@pytest.fixture() +def kmip(monkeypatch): + client = object.__new__(KMIPClient) + client.client = MagicMock() + kv = _FakeKVStore() + monkeypatch.setattr(KMIPClient, "_kv_store", staticmethod(lambda: kv)) + uids = iter(f"uid-{i}" for i in range(100)) + client.client.create.side_effect = lambda *a, **k: next(uids) + client.client.encrypt.side_effect = lambda data, **k: (bytes(b ^ 0x5A for b in data), b"\x01" * 16) + client.client.decrypt.side_effect = lambda data, **k: bytes(b ^ 0x5A for b in data) + return client, kv + + +class TestKMIPWrapping: + + def test_round_trip(self, kmip): + client, _ = kmip + client.ensure_key_encryption_key("k") + wrapped = client.wrap_secret("k", _fields(password="p"), CTX) + assert json.loads(wrapped)["iv"] == "01" * 16 + assert _plain(client.unwrap_secret("k", wrapped, CTX)) == {"password": "p"} + + def test_ensure_does_not_replace_existing_kek(self, kmip): + client, kv = kmip + client.ensure_key_encryption_key("k") + client.ensure_key_encryption_key("k") + assert client.client.create.call_count == 1 + assert kv.get(b"kms/kmip/kek/k") == b"uid-0" + + def test_cbc_bit_flip_is_caught_by_digest(self, kmip): + client, _ = kmip + client.ensure_key_encryption_key("k") + record = json.loads(client.wrap_secret("k", _fields(password="p"), CTX)) + ct = bytearray.fromhex(record["ciphertext"]) + ct[40] ^= 0x01 + record["ciphertext"] = ct.hex() + with pytest.raises(KMSException, match="integrity"): + client.unwrap_secret("k", json.dumps(record), CTX) + + def test_garbage_record_is_refused(self, kmip): + client, _ = kmip + client.ensure_key_encryption_key("k") + for bad in ("not json", "[]", '{"ciphertext": "zz"}'): + with pytest.raises(KMSException, match="not valid"): + client.unwrap_secret("k", bad, CTX) + + def test_get_secret_data(self, kmip): + client, _ = kmip + client.client.get.return_value = kmip_objects.SecretData( + b'{"password": "p"}', enums.SecretDataType.PASSWORD) + assert client.get_secret_data("u1") == b'{"password": "p"}' + + def test_get_secret_data_refuses_keys(self, kmip): + client, _ = kmip + client.client.get.return_value = kmip_objects.SymmetricKey( + enums.CryptographicAlgorithm.AES, 128, b"\x00" * 16) + with pytest.raises(KMSException, match="not SecretData"): + client.get_secret_data("u1") + + +@pytest.fixture() +def vault(): + client = object.__new__(HCPClient) + client.client = MagicMock() + client.transit_mount = "t" + client.kv_mount = "kv" + return client + + +class TestVault: + + def test_ensure_creates_only_when_missing(self, vault): + transit = vault.client.secrets.transit + transit.read_key.side_effect = hvac.exceptions.InvalidPath() + vault.ensure_key_encryption_key("k") + transit.create_key.assert_called_once() + + transit.reset_mock() + transit.read_key.side_effect = None + vault.ensure_key_encryption_key("k") + transit.create_key.assert_not_called() + + def test_wrap_uses_transit(self, vault): + transit = vault.client.secrets.transit + store = {} + transit.encrypt_data.side_effect = lambda name, plaintext, mount_point: ( + store.setdefault("pt", plaintext), {"data": {"ciphertext": "vault:v1:x"}})[1] + transit.decrypt_data.side_effect = lambda name, ciphertext, mount_point: { + "data": {"plaintext": store["pt"]}} + wrapped = vault.wrap_secret("k", _fields(password="p"), CTX) + assert wrapped == "vault:v1:x" + assert _plain(vault.unwrap_secret("k", wrapped, CTX)) == {"password": "p"} + + def test_read_kv_secret(self, vault): + vault.client.secrets.kv.v2.read_secret_version.return_value = { + "data": {"data": {"username": "u", "password": "p"}}} + assert _plain(vault.read_kv_secret("arrays", "pure01")) == { + "username": "u", "password": "p"} + + def test_read_kv_secret_refuses_non_string_fields_without_values(self, vault): + vault.client.secrets.kv.v2.read_secret_version.return_value = { + "data": {"data": {"password": "hunter2", "port": 443}}} + with pytest.raises(KMSException) as e: + vault.read_kv_secret("arrays", "pure01") + assert "port" in e.value.message and "hunter2" not in e.value.message + + def test_read_kv_secret_missing(self, vault): + vault.client.secrets.kv.v2.read_secret_version.side_effect = hvac.exceptions.InvalidPath() + with pytest.raises(KMSException, match="No secret"): + vault.read_kv_secret("arrays", "nope") + + +# -- CredentialProvider ----------------------------------------------------- + +def _cluster(kms_settings=None, allow_local=False): + cluster = Cluster() + cluster.uuid = "c1" + cluster.set_kms_settings(kms_settings) + cluster.allow_local_kms_credentials = allow_local + return cluster + + +VAULT = {"provider": "vault", "base_url": "https://vault.example:8200"} +KMIP = {"provider": "kmip", "server_url": "kmip.example:5696"} + + +class Clock: + def __init__(self): + self.t = 1000.0 + + def __call__(self): + return self.t + + +def _provider(cluster, kms, clock=None): + factory = MagicMock(return_value=kms) + provider = CredentialProvider(cluster, kms_factory=factory, clock=clock or Clock()) + return provider, factory + + +class TestOptIn: + + def test_local_kms_is_refused_without_opt_in(self): + provider, factory = _provider(_cluster(), FakeKMS()) + with pytest.raises(CredentialError, match="opt in"): + provider.store({"password": "p"}, CTX) + factory.assert_not_called() + + def test_local_kms_refuses_resolve_after_opt_in_is_withdrawn(self): + cluster = _cluster(allow_local=True) + provider, _ = _provider(cluster, FakeKMS()) + ref = provider.store({"password": "p"}, CTX) + cluster.allow_local_kms_credentials = False + with pytest.raises(CredentialError, match="opt in"): + CredentialProvider(cluster, kms_factory=MagicMock(return_value=FakeKMS())).resolve(ref, CTX) + + def test_local_kms_works_with_opt_in(self): + provider, _ = _provider(_cluster(allow_local=True), FakeKMS()) + ref = provider.store({"password": "p"}, CTX) + assert ref.backend == "local" + assert provider.resolve(ref, CTX).get("password").get_secret_value() == "p" + + def test_cluster_default_is_no_opt_in(self): + assert Cluster().allow_local_kms_credentials is False + + +class TestManaged: + + def test_store_and_resolve_round_trip(self): + kms = FakeKMS() + provider, factory = _provider(_cluster(VAULT), kms) + ref = provider.store({"username": "u", "password": SecretStr("hunter2")}, CTX) + assert ref == ManagedCredentialRef( + backend="vault", kek_name=kek_name_for("c1"), wrapped=ref.wrapped) + assert kms.created == ["credentials-c1"] + cred = provider.resolve(ref.model_dump(), CTX) # dict form, as stored in FDB + assert cred.get("password").get_secret_value() == "hunter2" + assert "hunter2" not in repr(cred) + + def test_kek_is_reused(self): + kms = FakeKMS() + provider, _ = _provider(_cluster(VAULT), kms) + provider.store({"a": "1"}, CTX) + provider.store({"a": "2"}, CTX) + assert kms.created == ["credentials-c1"] + + def test_other_context_is_refused(self): + provider, _ = _provider(_cluster(KMIP), FakeKMS()) + ref = provider.store({"password": "p"}, CTX) + with pytest.raises(CredentialError, match="different context"): + provider.resolve(ref, "array/a1/https://evil.example:443") + + def test_backend_change_is_reported(self): + kms = FakeKMS() + provider, _ = _provider(_cluster(VAULT), kms) + ref = provider.store({"password": "p"}, CTX) + provider2, _ = _provider(_cluster(KMIP), kms) + with pytest.raises(CredentialError, match="stored with the vault"): + provider2.resolve(ref, CTX) + + def test_empty_inputs_are_refused(self): + provider, _ = _provider(_cluster(VAULT), FakeKMS()) + with pytest.raises(CredentialError): + provider.store({}, CTX) + with pytest.raises(CredentialError): + provider.store({"a": "b"}, "") + + def test_credentials_cert_role_is_used(self): + settings = dict(VAULT, credentials_cert_role="simplyblock-credentials") + provider, factory = _provider(_cluster(settings), FakeKMS()) + provider.store({"a": "b"}, CTX) + assert factory.call_args.kwargs == {"cert_role": "simplyblock-credentials"} + + def test_kms_connection_failure_is_wrapped(self): + provider = CredentialProvider( + _cluster(VAULT), kms_factory=MagicMock(side_effect=KMSException("Authentication failed"))) + with pytest.raises(CredentialError, match="Authentication failed"): + provider.store({"a": "b"}, CTX) + + +class TestCache: + + def test_ttl_and_invalidate(self): + kms = FakeKMS() + clock = Clock() + provider, factory = _provider(_cluster(VAULT), kms, clock) + ref = provider.store({"a": "b"}, CTX) + factory.reset_mock() + + provider.resolve(ref, CTX) + provider.resolve(ref, CTX) + assert factory.call_count == 1 + + clock.t += 301 + provider.resolve(ref, CTX) + assert factory.call_count == 2 + + provider.invalidate(ref) + provider.resolve(ref, CTX) + assert factory.call_count == 3 + + provider.invalidate() + provider.resolve(ref, CTX) + assert factory.call_count == 4 + + def test_cache_is_per_context(self): + provider, _ = _provider(_cluster(VAULT), FakeKMS()) + ref = provider.store({"a": "b"}, CTX) + provider.resolve(ref, CTX) + with pytest.raises(CredentialError): + provider.resolve(ref, "other") + + +class TestExternal: + + def _vault_kms(self, data): + kms = MagicMock(spec=HCPClient) + kms.__enter__.return_value = kms + kms.read_kv_secret.return_value = {k: SecretStr(v) for k, v in data.items()} + return kms + + def _kmip_kms(self, raw): + kms = MagicMock(spec=KMIPClient) + kms.__enter__.return_value = kms + kms.get_secret_data.return_value = raw + return kms + + def test_vault_kv(self): + kms = self._vault_kms({"username": "u", "password": "p"}) + provider, _ = _provider(_cluster(VAULT), kms) + ref = VaultKVCredentialRef(mount="customer-kv", path="arrays/pure01", version=3) + cred = provider.resolve(ref, CTX) + assert cred.field_names == {"username", "password"} + kms.read_kv_secret.assert_called_once_with("customer-kv", "arrays/pure01", 3) + + def test_vault_kv_context_field(self): + provider, _ = _provider( + _cluster(VAULT), self._vault_kms({"password": "p", CONTEXT_FIELD: CTX})) + assert provider.resolve( + VaultKVCredentialRef(mount="m", path="p"), CTX).field_names == {"password"} + + provider, _ = _provider( + _cluster(VAULT), self._vault_kms({"password": "p", CONTEXT_FIELD: "other"})) + with pytest.raises(CredentialError, match="different context"): + provider.resolve(VaultKVCredentialRef(mount="m", path="p"), CTX) + + def test_vault_kv_needs_vault(self): + provider, _ = _provider(_cluster(KMIP), self._kmip_kms(b"{}")) + with pytest.raises(CredentialError, match="needs the cluster KMS to be Vault"): + provider.resolve(VaultKVCredentialRef(mount="m", path="p"), CTX) + + def test_kmip_json_object(self): + provider, _ = _provider( + _cluster(KMIP), self._kmip_kms(b'{"username": "u", "password": "p"}')) + cred = provider.resolve({"kind": "kmip-secret", "uid": "42"}, CTX) + assert cred.get("password").get_secret_value() == "p" + + def test_kmip_raw_value_as_field(self): + provider, _ = _provider(_cluster(KMIP), self._kmip_kms(b"s3cr3t")) + cred = provider.resolve(KMIPSecretCredentialRef(uid="42", field="api_token"), CTX) + assert cred.get("api_token").get_secret_value() == "s3cr3t" + + def test_kmip_raw_value_without_field_is_refused(self): + provider, _ = _provider(_cluster(KMIP), self._kmip_kms(b"s3cr3t")) + with pytest.raises(CredentialError, match="set 'field'") as e: + provider.resolve(KMIPSecretCredentialRef(uid="42"), CTX) + assert "s3cr3t" not in str(e.value) + + def test_kmip_needs_kmip(self): + provider, _ = _provider(_cluster(VAULT), self._vault_kms({"a": "b"})) + with pytest.raises(CredentialError, match="needs the cluster KMS to be KMIP"): + provider.resolve(KMIPSecretCredentialRef(uid="42"), CTX) + + def test_external_refs_refused_under_local_kms_without_opt_in(self): + provider, _ = _provider(_cluster(), FakeKMS()) + with pytest.raises(CredentialError, match="opt in"): + provider.resolve(VaultKVCredentialRef(mount="m", path="p"), CTX) + + +def test_credential_missing_field_names_fields_not_values(): + cred = Credential(_fields(username="u", password="hunter2")) + with pytest.raises(CredentialError) as e: + cred.get("token") + assert "password" in str(e.value) and "hunter2" not in str(e.value) + + +# -- cluster set: bool parsing --------------------------------------------- + +@pytest.mark.parametrize("raw, expected", [ + ("true", True), ("True", True), ("1", True), ("yes", True), + ("false", False), ("False", False), ("0", False), ("off", False), +]) +def test_cluster_set_parses_bools(monkeypatch, raw, expected): + from simplyblock_core import cluster_ops + + cluster = Cluster() + cluster.allow_local_kms_credentials = not expected + cluster.write_to_db = MagicMock() + db = MagicMock() + db.get_cluster_by_id.return_value = cluster + monkeypatch.setattr(cluster_ops, "db_controller", db) + + cluster_ops.set_("c1", "allow_local_kms_credentials", raw) + + assert cluster.allow_local_kms_credentials is expected + cluster.write_to_db.assert_called_once() + + +def test_cluster_set_rejects_non_bool(monkeypatch): + from simplyblock_core import cluster_ops + + cluster = Cluster() + cluster.write_to_db = MagicMock() + db = MagicMock() + db.get_cluster_by_id.return_value = cluster + monkeypatch.setattr(cluster_ops, "db_controller", db) + + with pytest.raises(ValueError): + cluster_ops.set_("c1", "allow_local_kms_credentials", "maybe") + cluster.write_to_db.assert_not_called() diff --git a/tests/unit/test_failure_domain.py b/tests/unit/test_failure_domain.py index ac7f3c6560..88034ab870 100644 --- a/tests/unit/test_failure_domain.py +++ b/tests/unit/test_failure_domain.py @@ -616,6 +616,10 @@ def setUp(self): self.snm = snm self._patches = [ patch.object(snm, "is_new_migrated_node", return_value=False), + # These 2-domain layouts test the FD suspend contract; they sit + # below the n+k failure-domain floor, which adds DEGRADED on its + # own (covered in test_cluster_floors). + patch.object(snm.floors, "evaluate", return_value=[]), ] for p in self._patches: p.start() diff --git a/tests/unit/test_ftt_protection.py b/tests/unit/test_ftt_protection.py index ddbdd07086..17b76202da 100644 --- a/tests/unit/test_ftt_protection.py +++ b/tests/unit/test_ftt_protection.py @@ -5,7 +5,10 @@ - FTT=1 (npcs=1): block if any node not online or journal replicating - FTT=2 (npcs=2, ft=2): allow one not-online, block at two - npcs=2, ft=1: like FTT=2 plus primary-secondary pair constraint - - Rebalancing: block based on headroom (active_nodes - ndcs - npcs) + - Resource floors (simplyblock_core.floors) are judged last: the removal is + refused when the cluster without the node would be below a floor. The + FTT tests below patch the floors out (their fixtures model the FTT rule + with as few nodes as it takes); TestFloorGate exercises the gate itself. - Journal replication counts as one additional not-online node - REMOVED and IN_CREATION nodes are excluded from counts """ @@ -13,7 +16,11 @@ import unittest from unittest.mock import MagicMock, patch +import pytest + +from simplyblock_core import storage_node_ops from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.nvme_device import NVMeDevice from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.rpc_client import RPCConnectionError from simplyblock_core.storage_node_ops import _check_ftt_allows_node_removal @@ -78,6 +85,30 @@ def _set_jm_replicating(node): rpc.jc_get_jm_status.return_value = {"jm1": False} +def _with_devices(node, count=1): + """Give a mock node ``count`` online data devices (the floor gate counts them).""" + devs = [] + for i in range(count): + d = MagicMock(spec=NVMeDevice) + d.status = NVMeDevice.STATUS_ONLINE + d.get_id = MagicMock(return_value=f"{node.get_id()}-dev-{i}") + devs.append(d) + node.nvme_devices = devs + return node + + +@pytest.fixture(autouse=True) +def _floors_out_of_the_ftt_tests(request): + """The FTT fixtures are sized for the FTT rule alone; the resource floors + (n+2k nodes, one data device per node, ...) are a separate gate with its + own tests (TestFloorGate), so they are patched out everywhere else.""" + if request.cls is not None and request.cls.__name__ == "TestFloorGate": + yield + return + with patch.object(storage_node_ops.floors, "evaluate", return_value=[]): + yield + + # --------------------------------------------------------------------------- # Non-HA clusters # --------------------------------------------------------------------------- @@ -278,101 +309,6 @@ def test_two_unrelated_offline_blocks(self): allowed, reason = _check_ftt_allows_node_removal("n1", db) self.assertFalse(allowed) self.assertIn("npcs=2/ft=1", reason) - - -# --------------------------------------------------------------------------- -# Rebalancing -# --------------------------------------------------------------------------- - -class TestRebalancing(unittest.TestCase): - - def test_rebalancing_exact_minimum_nodes_blocks(self): - """ndcs=2, npcs=1: minimum=3. With exactly 3 nodes and rebalancing → block.""" - cl = _cluster(npcs=1, ndcs=2, ft=1, rebalancing=True) - nodes = [_node("n1"), _node("n2"), _node("n3")] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n1", db) - self.assertFalse(allowed) - self.assertIn("rebalancing", reason) - - def test_rebalancing_one_above_minimum_allows(self): - """ndcs=2, npcs=1: minimum=3. With 4 nodes and rebalancing → allowed.""" - cl = _cluster(npcs=1, ndcs=2, ft=1, rebalancing=True) - nodes = [_node("n1"), _node("n2"), _node("n3"), _node("n4")] - db = _db(cl, nodes) - allowed, _ = _check_ftt_allows_node_removal("n1", db) - self.assertTrue(allowed) - - def test_rebalancing_one_above_min_but_one_offline_blocks(self): - """ndcs=2, npcs=1: 4 nodes, 1 offline, rebalancing → headroom=1, not_online=1 → block.""" - cl = _cluster(npcs=1, ndcs=2, ft=1, rebalancing=True) - nodes = [_node("n1"), _node("n2"), _node("n3"), - _node("n4", status=StorageNode.STATUS_OFFLINE)] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n1", db) - self.assertFalse(allowed) - self.assertIn("rebalancing", reason) - - def test_rebalancing_two_above_min_one_offline_allows_ftt2(self): - """ndcs=2, npcs=2, ft=2: 6 nodes, 1 offline, rebalancing → headroom=2, not_online=1 → allowed.""" - cl = _cluster(npcs=2, ndcs=2, ft=2, rebalancing=True) - nodes = [_node("n1"), _node("n2"), _node("n3"), _node("n4"), - _node("n5"), _node("n6", status=StorageNode.STATUS_OFFLINE)] - db = _db(cl, nodes) - allowed, _ = _check_ftt_allows_node_removal("n1", db) - self.assertTrue(allowed) - - def test_rebalancing_npcs2_exact_minimum_blocks(self): - """ndcs=2, npcs=2: minimum=4. With exactly 4 nodes and rebalancing → block.""" - cl = _cluster(npcs=2, ndcs=2, ft=2, rebalancing=True) - nodes = [_node("n1"), _node("n2"), _node("n3"), _node("n4")] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n1", db) - self.assertFalse(allowed) - self.assertIn("rebalancing", reason) - - def test_rebalancing_npcs2_one_above_allows_first(self): - """ndcs=2, npcs=2: 5 nodes, rebalancing → headroom=1, not_online=0 → allowed.""" - cl = _cluster(npcs=2, ndcs=2, ft=2, rebalancing=True) - nodes = [_node(f"n{i}") for i in range(5)] - db = _db(cl, nodes) - allowed, _ = _check_ftt_allows_node_removal("n0", db) - self.assertTrue(allowed) - - def test_rebalancing_npcs2_one_above_blocks_second(self): - """ndcs=2, npcs=2: 5 nodes, 1 offline, rebalancing → headroom=1, not_online=1 → block.""" - cl = _cluster(npcs=2, ndcs=2, ft=2, rebalancing=True) - nodes = [_node("n1"), _node("n2"), _node("n3"), _node("n4"), - _node("n5", status=StorageNode.STATUS_OFFLINE)] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n1", db) - self.assertFalse(allowed) - self.assertIn("rebalancing", reason) - - def test_not_rebalancing_exact_minimum_allows(self): - """Same scenario but NOT rebalancing → rebalancing check doesn't apply.""" - cl = _cluster(npcs=1, ndcs=2, ft=1, rebalancing=False) - nodes = [_node("n1"), _node("n2"), _node("n3")] - db = _db(cl, nodes) - allowed, _ = _check_ftt_allows_node_removal("n1", db) - self.assertTrue(allowed) - - def test_rebalancing_removed_nodes_excluded(self): - """Removed nodes don't count as active, reducing headroom.""" - cl = _cluster(npcs=1, ndcs=2, ft=1, rebalancing=True) - # 4 nodes but one is removed → 3 active → exact minimum → block - nodes = [_node("n1"), _node("n2"), _node("n3"), - _node("n4", status=StorageNode.STATUS_REMOVED)] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n1", db) - self.assertFalse(allowed) - self.assertIn("rebalancing", reason) - - -# --------------------------------------------------------------------------- -# Edge cases -# --------------------------------------------------------------------------- - class TestEdgeCases(unittest.TestCase): def test_node_not_found(self): @@ -849,73 +785,6 @@ def test_fd_disabled_ignores_domains_uses_node_count(self): allowed, reason = _check_ftt_allows_node_removal("n1", db) self.assertFalse(allowed) self.assertIn("FTT=1", reason) - - -# --------------------------------------------------------------------------- -# Rebalancing additional scenarios -# --------------------------------------------------------------------------- - -class TestRebalancingAdditional(unittest.TestCase): - - def test_rebalancing_ndcs4_npcs2_six_nodes_blocks(self): - """ndcs=4, npcs=2: minimum=6. With 6 nodes, rebalancing → headroom=0 → block.""" - cl = _cluster(npcs=2, ndcs=4, ft=2, rebalancing=True) - nodes = [_node(f"n{i}") for i in range(6)] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n0", db) - self.assertFalse(allowed) - self.assertIn("rebalancing", reason) - - def test_rebalancing_ndcs4_npcs2_seven_nodes_allows(self): - """ndcs=4, npcs=2: minimum=6. With 7 nodes, rebalancing → headroom=1 → allowed.""" - cl = _cluster(npcs=2, ndcs=4, ft=2, rebalancing=True) - nodes = [_node(f"n{i}") for i in range(7)] - db = _db(cl, nodes) - allowed, _ = _check_ftt_allows_node_removal("n0", db) - self.assertTrue(allowed) - - def test_rebalancing_ndcs1_npcs1_two_nodes_blocks(self): - """ndcs=1, npcs=1: minimum=2. With 2 nodes, rebalancing → block.""" - cl = _cluster(npcs=1, ndcs=1, ft=1, rebalancing=True) - nodes = [_node("n1"), _node("n2")] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n1", db) - self.assertFalse(allowed) - self.assertIn("rebalancing", reason) - - def test_rebalancing_ndcs1_npcs1_three_nodes_allows(self): - """ndcs=1, npcs=1: minimum=2. With 3 nodes, rebalancing → allowed.""" - cl = _cluster(npcs=1, ndcs=1, ft=1, rebalancing=True) - nodes = [_node("n1"), _node("n2"), _node("n3")] - db = _db(cl, nodes) - allowed, _ = _check_ftt_allows_node_removal("n1", db) - self.assertTrue(allowed) - - def test_rebalancing_in_creation_not_counted_as_active(self): - """in_creation nodes reduce active count, reducing headroom.""" - cl = _cluster(npcs=1, ndcs=2, ft=1, rebalancing=True) - # 4 nodes but 1 in_creation → 3 active → minimum → block - nodes = [_node("n1"), _node("n2"), _node("n3"), - _node("n4", status=StorageNode.STATUS_IN_CREATION)] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n1", db) - self.assertFalse(allowed) - - def test_rebalancing_check_happens_before_ftt_check(self): - """Even if FTT would allow it, rebalancing with no headroom blocks.""" - cl = _cluster(npcs=2, ndcs=2, ft=2, rebalancing=True) - # 4 nodes, all online, FTT=2 would allow but rebalancing blocks (headroom=0) - nodes = [_node(f"n{i}") for i in range(4)] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n0", db) - self.assertFalse(allowed) - self.assertIn("rebalancing", reason) - - -# --------------------------------------------------------------------------- -# Reason string validation -# --------------------------------------------------------------------------- - class TestReasonStrings(unittest.TestCase): def test_ftt1_reason_includes_count(self): @@ -939,15 +808,6 @@ def test_ftt2_reason_includes_count(self): self.assertFalse(allowed) self.assertIn("2 not-online", reason) - def test_rebalancing_reason_includes_node_counts(self): - cl = _cluster(npcs=1, ndcs=2, ft=1, rebalancing=True) - nodes = [_node("n1"), _node("n2"), _node("n3")] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n1", db) - self.assertFalse(allowed) - self.assertIn("3 active nodes", reason) - self.assertIn("rebalancing", reason) - def test_pair_reason_includes_node_ids(self): cl = _cluster(npcs=2, ft=1) nodes = [_node("n1", secondary_id="n2"), @@ -1099,3 +959,85 @@ def test_tertiary_counts_as_own_secondary(self): if __name__ == "__main__": unittest.main() + + +# --------------------------------------------------------------------------- +# Resource floors: the cluster after the removal must stay at or above them +# --------------------------------------------------------------------------- + +class TestFloorGate(unittest.TestCase): + """2+1: node floor n+2k = 4, data-device floor 4 (n+k = 3 with + raid_devices), one data device per node.""" + + def _nodes(self, count, devices=1): + return [_with_devices(_node(f"n{i}"), devices) for i in range(count)] + + def test_removal_that_leaves_the_node_floor_is_allowed(self): + cl = _cluster(npcs=1, ndcs=2, ft=1) + db = _db(cl, self._nodes(5, devices=2)) + allowed, reason = _check_ftt_allows_node_removal("n1", db) + self.assertTrue(allowed, reason) + + def test_removal_below_the_node_floor_is_refused_and_named(self): + cl = _cluster(npcs=1, ndcs=2, ft=1) + db = _db(cl, self._nodes(4, devices=2)) + allowed, reason = _check_ftt_allows_node_removal("n1", db) + self.assertFalse(allowed) + self.assertIn("nodes online 3 < floor 4 (n+2k)", reason) + + def test_removal_below_the_data_device_floor_is_refused(self): + cl = _cluster(npcs=1, ndcs=2, ft=1) + nodes = self._nodes(5, devices=1) + # Without n1 four devices remain -- the floor. Take one away. + nodes[4].nvme_devices = [] + db = _db(cl, nodes) + allowed, reason = _check_ftt_allows_node_removal("n1", db) + self.assertFalse(allowed) + self.assertIn("node n4 has 0 online data devices < floor 1", reason) + self.assertIn("data devices online 3 < floor 4 (n+2k)", reason) + + def test_raid_devices_lowers_the_data_device_floor(self): + cl = _cluster(npcs=1, ndcs=2, ft=1) + cl.raid_devices = True + nodes = self._nodes(5, devices=1) + nodes[4].nvme_devices = [] + db = _db(cl, nodes) + allowed, reason = _check_ftt_allows_node_removal("n1", db) + # The per-node floor still refuses; the cluster-wide one no longer does. + self.assertFalse(allowed) + self.assertIn("node n4 has 0 online data devices", reason) + self.assertNotIn("data devices online", reason) + + def test_journal_device_does_not_count(self): + cl = _cluster(npcs=1, ndcs=2, ft=1) + nodes = self._nodes(5, devices=1) + jm = MagicMock(spec=NVMeDevice) + jm.status = NVMeDevice.STATUS_JM + jm.get_id = MagicMock(return_value="n4-jm") + nodes[4].nvme_devices = [jm] + db = _db(cl, nodes) + allowed, reason = _check_ftt_allows_node_removal("n1", db) + self.assertFalse(allowed) + self.assertIn("node n4 has 0 online data devices", reason) + + def test_rebalancing_no_longer_adds_a_headroom_rule(self): + """The former headroom formula (active nodes - (ndcs+npcs)) is gone: + with the floors satisfied a rebalance does not block the removal.""" + cl = _cluster(npcs=1, ndcs=2, ft=1, rebalancing=True) + db = _db(cl, self._nodes(5, devices=2)) + allowed, reason = _check_ftt_allows_node_removal("n1", db) + self.assertTrue(allowed, reason) + + def test_failure_domain_floor(self): + cl = _cluster(npcs=1, ndcs=2, ft=1, enable_failure_domain=True) + # Six nodes in three domains: removing n0 leaves domains {0,1,2} = n+k. + nodes = [_with_devices(_node(f"n{i}", failure_domain=i % 3), 1) for i in range(6)] + db = _db(cl, nodes) + allowed, reason = _check_ftt_allows_node_removal("n0", db) + self.assertTrue(allowed, reason) + # Two domains only: below the n+k = 3 domain floor. + nodes = [_with_devices(_node(f"n{i}", failure_domain=i % 2), 1) for i in range(6)] + db = _db(cl, nodes) + allowed, reason = _check_ftt_allows_node_removal("n0", db) + self.assertFalse(allowed) + self.assertIn("failure domains online 2 < floor 3 (n+k)", reason) diff --git a/tests/unit/test_lblk_aio_block_size.py b/tests/unit/test_lblk_aio_block_size.py new file mode 100644 index 0000000000..14103e099b --- /dev/null +++ b/tests/unit/test_lblk_aio_block_size.py @@ -0,0 +1,389 @@ +"""Unit tests for the block size of lblk AIO base bdevs. + +A new device gets a 4096-byte AIO bdev unless 4096 is impossible, then the +device's logical block size. A device that already has a record is always +re-created with the block size it was formatted with, because alceml's +on-disk layout depends on it: 0 (legacy, created before the block size was +recorded) means "no block size", exactly as it was created. + +Covered: + - utils.choose_aio_block_size: 512e, 512n with a non-4K size, 4Kn, logical + above 4096, an agent that does not send logical_block_size, partition + start alignment. + - utils.addAioDevices: new device -> chosen size recorded on the device and + on the node's lblk_devices entry; legacy record -> no block size; 4096 + record -> 4096 whatever the inventory says; record dropped by an older + control-plane service -> recovered from the lblk_devices entry; journal + device record; SPDK refusing 4096 for a new device -> retried without; + block-size mismatch -> refused. + - utils.resolve_lblk_entries carries the block-size fields. + - storage_node_ops._reconcile_discovered_devices restores a dropped size. + - device_controller.restart_device / new_device_from_failed pass the + recorded size. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import utils +from simplyblock_core.controllers import device_controller +from simplyblock_core.models.nvme_device import JMDevice, NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.storage_node_ops import _reconcile_discovered_devices + +GIB = 1 << 30 + + +def _entry(serial="S1", size=100 * GIB, logical=512, physical=4096, **extra): + e = {"name": "sdb", "serial": serial, "by_id": f"/dev/disk/by-id/wwn-{serial}", + "size": size, "numa": 0, "model": "M", "current_path": "/dev/sdb", + "has_partitions": False} + if logical is not None: + e["logical_block_size"] = logical + if physical is not None: + e["physical_block_size"] = physical + e.update(extra) + return e + + +def _snode(devices=(), lblk_devices=None, jm_device=None): + n = StorageNode() + n.uuid = "node-1" + n.cluster_id = "cluster-1" + n.nvme_devices = list(devices) + n.lblk_devices = lblk_devices if lblk_devices is not None else [ + {"name": "sdb", "serial": "S1", "size": 100 * GIB}] + if jm_device is not None: + n.jm_device = jm_device + return n + + +def _record(serial="S1", aio_block_size=0, status=NVMeDevice.STATUS_ONLINE): + d = NVMeDevice() + d.uuid = f"dev-{serial}" + d.serial_number = serial + d.bdev_type = "aio" + d.nvme_bdev = utils.aio_bdev_name_for_serial(serial) + d.aio_block_size = aio_block_size + d.status = status + return d + + +class _FakeRpc: + """SPDK fake modelling bdev_aio_create's block-size rules: no block size + -> the logical block size; one below the logical block size -> refused.""" + + def __init__(self, logical=512, num_bytes=100 * GIB, existing=None, + ignore_block_size=False): + self.host = "test-host" + self.logical = logical + self.num_bytes = num_bytes + self.bdevs = dict(existing or {}) + self.ignore_block_size = ignore_block_size + self.creates = [] + + def bdev_get(self, name): + return self.bdevs.get(name) + + def bdev_aio_create(self, name, filename, block_size=0): + self.creates.append(block_size) + if block_size and block_size < self.logical: + return None + bs = self.logical if (not block_size or self.ignore_block_size) else block_size + self.bdevs[name] = {"name": name, "block_size": bs, + "num_blocks": self.num_bytes // bs} + return name + + def bdev_examine(self, name): + return True + + def bdev_wait_for_examine(self): + return True + + def bdev_set_qd_sampling_period(self, name, period): + return True + + +class TestChooseAioBlockSize(unittest.TestCase): + + def test_512e_gets_4096(self): + self.assertEqual(utils.choose_aio_block_size(_entry(logical=512, physical=4096)), 4096) + + def test_512n_with_non_4k_size_gets_512(self): + self.assertEqual(utils.choose_aio_block_size( + _entry(logical=512, physical=512, size=100 * GIB + 512)), 512) + + def test_512n_with_4k_size_gets_4096(self): + self.assertEqual(utils.choose_aio_block_size(_entry(logical=512, physical=512)), 4096) + + def test_4kn_gets_4096(self): + self.assertEqual(utils.choose_aio_block_size(_entry(logical=4096, physical=4096)), 4096) + + def test_logical_above_4096_kept(self): + self.assertEqual(utils.choose_aio_block_size(_entry(logical=8192)), 8192) + + def test_agent_without_field_4k_size_gets_4096(self): + self.assertEqual(utils.choose_aio_block_size(_entry(logical=None, physical=None)), 4096) + + def test_agent_without_field_odd_size_leaves_it_to_spdk(self): + self.assertEqual(utils.choose_aio_block_size( + _entry(logical=None, physical=None, size=100 * GIB + 512)), 0) + + def test_unaligned_partition_start_gets_logical(self): + self.assertEqual(utils.choose_aio_block_size( + _entry(type="part", start_bytes=63 * 512)), 512) + + def test_aligned_partition_start_gets_4096(self): + self.assertEqual(utils.choose_aio_block_size( + _entry(type="part", start_bytes=2048 * 512)), 4096) + + +class TestAddAioDevicesBlockSize(unittest.TestCase): + + def test_new_512e_device_created_at_4096_and_recorded(self): + rpc = _FakeRpc(logical=512) + snode = _snode() + devs = utils.addAioDevices(rpc, snode, [_entry()]) + self.assertEqual(rpc.creates, [4096]) + self.assertEqual(devs[0].aio_block_size, 4096) + self.assertEqual(snode.lblk_devices[0]["aio_block_size"], 4096) + + def test_new_512n_odd_size_created_at_512(self): + rpc = _FakeRpc(logical=512, num_bytes=100 * GIB + 512) + snode = _snode() + devs = utils.addAioDevices(rpc, snode, [_entry(physical=512, size=100 * GIB + 512)]) + self.assertEqual(rpc.creates, [512]) + self.assertEqual(devs[0].aio_block_size, 512) + self.assertEqual(snode.lblk_devices[0]["aio_block_size"], 512) + + def test_new_4kn_device_created_at_4096(self): + rpc = _FakeRpc(logical=4096) + devs = utils.addAioDevices(rpc, _snode(), [_entry(logical=4096)]) + self.assertEqual(rpc.creates, [4096]) + self.assertEqual(devs[0].aio_block_size, 4096) + + def test_legacy_record_restarted_without_block_size(self): + rpc = _FakeRpc(logical=512) + snode = _snode(devices=[_record(aio_block_size=0)]) + devs = utils.addAioDevices(rpc, snode, [_entry()]) + self.assertEqual(rpc.creates, [0]) + self.assertEqual(devs[0].aio_block_size, 0) + self.assertNotIn("aio_block_size", snode.lblk_devices[0]) + + def test_4096_record_restarted_at_4096(self): + rpc = _FakeRpc(logical=512) + snode = _snode(devices=[_record(aio_block_size=4096)]) + # Whatever the inventory says now, the recorded size wins. + utils.addAioDevices(rpc, snode, [_entry(size=100 * GIB + 512)]) + self.assertEqual(rpc.creates, [4096]) + + def test_512_record_restarted_at_512(self): + rpc = _FakeRpc(logical=512) + snode = _snode(devices=[_record(aio_block_size=512)]) + utils.addAioDevices(rpc, snode, [_entry()]) + self.assertEqual(rpc.creates, [512]) + + def test_size_dropped_from_record_recovered_from_lblk_entry(self): + # An older control-plane service rewrote the node and dropped + # NVMeDevice.aio_block_size; the lblk_devices dict kept it. + rpc = _FakeRpc(logical=512) + snode = _snode(devices=[_record(aio_block_size=0)], + lblk_devices=[{"name": "sdb", "serial": "S1", "aio_block_size": 4096}]) + devs = utils.addAioDevices(rpc, snode, [_entry()]) + self.assertEqual(rpc.creates, [4096]) + self.assertEqual(devs[0].aio_block_size, 4096) + + def test_journal_device_record_used(self): + jm = JMDevice() + jm.serial_number = "S1" + jm.device_data_dict = {"serial_number": "S1", "aio_block_size": 4096} + rpc = _FakeRpc(logical=512) + utils.addAioDevices(rpc, _snode(jm_device=jm), [_entry(size=100 * GIB + 512)]) + self.assertEqual(rpc.creates, [4096]) + + def test_legacy_journal_device_record_without_block_size(self): + jm = JMDevice() + jm.serial_number = "S1" + jm.device_data_dict = {"serial_number": "S1"} + rpc = _FakeRpc(logical=512) + utils.addAioDevices(rpc, _snode(jm_device=jm), [_entry()]) + self.assertEqual(rpc.creates, [0]) + + def test_agent_without_field_new_device(self): + rpc = _FakeRpc(logical=512) + devs = utils.addAioDevices(rpc, _snode(), [_entry(logical=None, physical=None)]) + self.assertEqual(rpc.creates, [4096]) + self.assertEqual(devs[0].aio_block_size, 4096) + + def test_new_device_refused_4096_retried_without(self): + rpc = _FakeRpc(logical=8192) + snode = _snode() + devs = utils.addAioDevices(rpc, snode, [_entry(logical=None, physical=None)]) + self.assertEqual(rpc.creates, [4096, 0]) + self.assertEqual(devs[0].aio_block_size, 8192) + self.assertEqual(snode.lblk_devices[0]["aio_block_size"], 8192) + + def test_recorded_device_refused_is_not_retried(self): + rpc = _FakeRpc(logical=8192) + snode = _snode(devices=[_record(aio_block_size=4096)]) + with self.assertRaisesRegex(Exception, "bdev_aio_create failed"): + utils.addAioDevices(rpc, snode, [_entry()]) + self.assertEqual(rpc.creates, [4096]) + + def test_spdk_ignoring_block_size_refused(self): + rpc = _FakeRpc(logical=512, ignore_block_size=True) + snode = _snode(devices=[_record(aio_block_size=4096)]) + with self.assertRaisesRegex(Exception, "expected 4096"): + utils.addAioDevices(rpc, snode, [_entry()]) + + def test_existing_bdev_of_other_size_refused(self): + name = utils.aio_bdev_name_for_serial("S1") + rpc = _FakeRpc(existing={name: {"name": name, "block_size": 512, "num_blocks": 10}}) + snode = _snode(devices=[_record(aio_block_size=4096)]) + with self.assertRaisesRegex(Exception, "expected 4096"): + utils.addAioDevices(rpc, snode, [_entry()]) + self.assertEqual(rpc.creates, []) + + def test_existing_bdev_of_new_device_recorded_at_its_size(self): + name = utils.aio_bdev_name_for_serial("S1") + rpc = _FakeRpc(existing={name: {"name": name, "block_size": 512, "num_blocks": 10}}) + devs = utils.addAioDevices(rpc, _snode(), [_entry()]) + self.assertEqual(rpc.creates, []) + self.assertEqual(devs[0].aio_block_size, 512) + + +class TestResolveCarriesBlockSize(unittest.TestCase): + + def _live(self, serial): + return {"name": "sdx", "device_path": "/dev/sdx", "serial": serial, + "by_id_path": "", "size": 100 * GIB, "numa_node": 0, + "logical_block_size": 512, "physical_block_size": 4096, + "type": "disk"} + + def test_fields_carried_for_same_device(self): + configured = [{"name": "sdb", "serial": "S1", "size": 1, "aio_block_size": 4096}] + resolved, _ = utils.resolve_lblk_entries(configured, [self._live("S1")]) + self.assertEqual(resolved[0]["logical_block_size"], 512) + self.assertEqual(resolved[0]["physical_block_size"], 4096) + self.assertEqual(resolved[0]["aio_block_size"], 4096) + + def test_recorded_size_not_carried_to_a_different_device(self): + configured = [{"name": "sdx", "serial": "S1", "size": 1, "aio_block_size": 512}] + resolved, _ = utils.resolve_lblk_entries(configured, [self._live("OTHER")]) + self.assertNotIn("aio_block_size", resolved[0]) + + +class TestReconcileRestoresBlockSize(unittest.TestCase): + + def setUp(self): + # Not a multi-attach cluster: reconcile's foreign-LUN lookup (a DB + # read, multi-attach only) finds nothing. + p = patch("simplyblock_core.storage_node_ops._foreign_lun_keys", return_value=set()) + p.start() + self.addCleanup(p.stop) + + def test_dropped_size_restored(self): + db_dev = _record(aio_block_size=0) + found = _record(aio_block_size=4096) + snode = _snode(devices=[db_dev]) + _reconcile_discovered_devices(snode, [found]) + self.assertEqual(db_dev.aio_block_size, 4096) + + def test_recorded_size_never_overwritten(self): + db_dev = _record(aio_block_size=512) + found = _record(aio_block_size=4096) + _reconcile_discovered_devices(_snode(devices=[db_dev]), [found]) + self.assertEqual(db_dev.aio_block_size, 512) + + +class TestDeviceControllerBlockSize(unittest.TestCase): + + def _restart(self, aio_block_size, lblk_devices=None): + device = _record(aio_block_size=aio_block_size, status=NVMeDevice.STATUS_REMOVED) + device.node_id = "node-1" + device.cluster_id = "cluster-1" + device.by_id_path = "/dev/disk/by-id/wwn-1" + device.nvmf_nqn = "" + device.alceml_bdev = "" + snode = MagicMock() + snode.cluster_id = "cluster-1" + snode.nvme_devices = [device] + snode.jm_device = None + snode.lblk_devices = lblk_devices or [] + + rpc = MagicMock() + state = {"present": False} + rpc.bdev_get.side_effect = ( + lambda name: {"name": name, "block_size": state.get("bs", 512)} + if state["present"] else None) + + def _aio_create(name, filename, block_size=0): + state["present"] = True + state["bs"] = block_size or 512 + return name + + rpc.bdev_aio_create.side_effect = _aio_create + snode.rpc_client.return_value = rpc + snode.client.return_value.get_blockdevices.return_value = ([], None) + + db = MagicMock() + db.get_storage_device_by_id.return_value = device + db.get_storage_node_by_id.return_value = snode + with patch.object(device_controller, "DBController", return_value=db), \ + patch.object(device_controller.tasks_controller, + "get_active_dev_restart_task", return_value=None), \ + patch.object(device_controller, "device_set_retries_exhausted"), \ + patch.object(device_controller, "device_set_unavailable"), \ + patch.object(device_controller, "_def_create_device_stack", return_value=True), \ + patch.object(device_controller, "device_set_io_error"), \ + patch.object(device_controller, "device_set_online"), \ + patch.object(device_controller, "device_events"): + result = device_controller.restart_device(device.uuid) + return result, rpc + + def test_restart_legacy_device_without_block_size(self): + result, rpc = self._restart(0) + self.assertTrue(result) + rpc.bdev_aio_create.assert_called_once_with("aio_S1", "/dev/disk/by-id/wwn-1") + + def test_restart_4096_device_at_4096(self): + result, rpc = self._restart(4096) + self.assertTrue(result) + rpc.bdev_aio_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-1", block_size=4096) + + def test_restart_uses_lblk_entry_when_record_dropped_it(self): + result, rpc = self._restart(0, lblk_devices=[{"serial": "S1", "aio_block_size": 4096}]) + self.assertTrue(result) + rpc.bdev_aio_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-1", block_size=4096) + + def test_new_device_from_failed_keeps_block_size(self): + device = _record(aio_block_size=4096, status=NVMeDevice.STATUS_FAILED_AND_MIGRATED) + device.by_id_path = "/dev/disk/by-id/wwn-1" + node = MagicMock() + node.get_id.return_value = "node-1" + node.nvme_devices = [device] + node.lblk_devices = [] + rpc = MagicMock() + state = {"present": False} + rpc.bdev_get.side_effect = ( + lambda name: {"name": name, "block_size": 4096} if state["present"] else None) + + def _aio_create(name, filename, block_size=0): + state["present"] = True + return name + + rpc.bdev_aio_create.side_effect = _aio_create + node.rpc_client.return_value = rpc + node.client.return_value.get_blockdevices.return_value = ([], None) + db = MagicMock() + db.get_storage_node_by_device_id.return_value = node + with patch.object(device_controller, "DBController", return_value=db): + self.assertTrue(device_controller.new_device_from_failed(device.uuid)) + rpc.bdev_aio_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-1", block_size=4096) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_lblk_device_resize.py b/tests/unit/test_lblk_device_resize.py index 9224666bdc..7de016121a 100644 --- a/tests/unit/test_lblk_device_resize.py +++ b/tests/unit/test_lblk_device_resize.py @@ -1,16 +1,18 @@ -"""Unit tests for the online growth of lblk devices (a LUN grown on its -storage array): device_controller.resize_device / grow_cluster_devices, the -frozen placement weight in distr_controller, and the host-side rescan in +"""Unit tests for the online growth of lblk devices (LUNs grown on their +storage array): device_controller.grow_cluster_devices, the frozen placement +weight in distr_controller, the capacity cap, and the host-side rescan in simplyblock_web.node_utils. Covered: - The cluster-map weight of a grown device stays at its placement size, - on full-map builds and on the add-device path. - - resize_device runs host rescan -> resize_begin -> bdev_aio_rescan -> - resize_end, persists size and placement_size, and reports uniformity. - - Refusals: not lblk, not online, partition, testing bdev, shrink. - - Recovery: a failed AIO rescan reopens alceml and leaves the device - unchanged; a failed reopen fails the I/O and sets the device unavailable. + on full-map builds and on the add-device path; growth never changes it. + - Growth is uniform: every data device grows by the smallest growth factor + of all block devices (1.5/2.0 -> 1.5); a device that did not grow stops + all growth; repeated runs change nothing. + - A shrunk block device is refused, logged, and raises device_shrunk. + - Refusals: not lblk, offline device or node, partition, testing bdev. + - Recovery: a failed reopen or AIO rescan leaves every size unchanged. + - The capacity statistics leave out the unused rest of a block device. - rescan_block_device: SCSI rescan, NVMe controller rescan, partitions. """ @@ -20,7 +22,7 @@ from unittest.mock import MagicMock, call, patch from simplyblock_core import constants, distr_controller, storage_node_ops, utils -from simplyblock_core.controllers import device_controller +from simplyblock_core.controllers import alerts_controller, device_controller from simplyblock_core.models.nvme_device import NVMeDevice from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.rpc_client import RPCRemoteError @@ -98,22 +100,20 @@ def test_add_device_weight_uses_placement_size(self): self.assertEqual(sent["devices"][0]["weight"], utils.convert_size(100 * _GIB, "GiB")) -class _ResizeHarness: - """Wires resize_device against mocks: one lblk cluster, one node.""" +class _GrowHarness: + """Wires grow_cluster_devices against mocks: one lblk cluster, one node + with data devices of 100 GiB whose block devices report the given sizes.""" - def __init__(self, dev, new_size, others=(), node_status=StorageNode.STATUS_ONLINE, - device_mode=constants.DEVICE_MODE_LBLK): - self.dev = dev - self.node = _node([dev, *others], status=node_status) + def __init__(self, sizes_gib, others=(), node_status=StorageNode.STATUS_ONLINE, + device_mode=constants.DEVICE_MODE_LBLK, dev_gib=100): + self.devs = [_aio_dev(f"dev-{i}", size_gib=dev_gib, order=i) for i in range(len(sizes_gib))] + self.node = _node([*self.devs, *others], status=node_status) self.cluster = MagicMock() self.cluster.device_mode = device_mode - self.cluster.status = "active" - self.cluster.reweight_pending = False self.rpc = MagicMock() self.node.rpc_client.return_value = self.rpc self.client = MagicMock() - self.client.rescan_block_device.return_value = ( - {"device": "sdb", "size": new_size, "rescanned": ["/sys/block/sdb/device/rescan"]}, None) + self.set_sizes(sizes_gib) self.node.client.return_value = self.client self.db = MagicMock() self.db.get_storage_device_by_id.side_effect = lambda _id: next( @@ -121,280 +121,278 @@ def __init__(self, dev, new_size, others=(), node_status=StorageNode.STATUS_ONLI self.db.get_storage_node_by_id.return_value = self.node self.db.get_storage_nodes_by_cluster_id.return_value = [self.node] self.db.get_cluster_by_id.return_value = self.cluster - self.db.get_job_tasks.return_value = [] - self.push_ok = True def _atomic(node, mut): mut(node) return node self.db.atomic_update.side_effect = _atomic - def run(self, fn=None, *args): + def set_sizes(self, sizes_gib): + by_path = {d.by_id_path: int(g * _GIB) for d, g in zip(self.devs, sizes_gib)} + self.client.rescan_block_device.side_effect = lambda path: ( + {"device": "sdx", "size": by_path[path], "rescanned": []}, None) + + def grow(self): + self.rpc.reset_mock() with patch.object(device_controller, "DBController", return_value=self.db), \ patch.object(device_controller.tasks_controller, "get_active_dev_restart_task", return_value=None), \ patch.object(device_controller, "device_set_unavailable") as self.set_unavail, \ patch.object(device_controller, "device_events") as self.events, \ - patch.object(device_controller.distr_controller, "send_cluster_map_to_node", - side_effect=lambda n: self.push_ok) as self.push, \ - patch.object(device_controller.tasks_controller, "add_device_mig_task_for_node", - return_value=True) as self.rebalance_task, \ patch.object(device_controller.time, "sleep"): - return (fn or device_controller.resize_device)(*(args or (self.dev.get_id(),))) - - -class TestResizeDevice(unittest.TestCase): - - def test_grows_device_and_freezes_weight(self): - h = _ResizeHarness(_aio_dev(size_gib=100), 200 * _GIB) - res = h.run() - self.assertTrue(res["resized"]) - self.assertEqual(res["old_size"], 100 * _GIB) - self.assertEqual(res["new_size"], 200 * _GIB) - self.assertEqual(h.dev.size, 200 * _GIB) - self.assertEqual(h.dev.placement_size, 100 * _GIB) - self.assertEqual(res["placement_size"], 100 * _GIB) - self.assertTrue(res["uniform"]) - h.client.rescan_block_device.assert_called_once_with("/dev/disk/by-id/wwn-dev-1") + return device_controller.grow_cluster_devices("cluster-1") + + def sizes(self): + return [d.size // _GIB for d in self.devs] + + +class TestGrowClusterDevices(unittest.TestCase): + + def test_uniform_growth_grows_every_device(self): + h = _GrowHarness([150, 150, 150]) + res = h.grow() + self.assertTrue(res["grown"]) + self.assertAlmostEqual(res["factor"], 1.5) + self.assertEqual(h.sizes(), [150, 150, 150]) + for d in h.devs: + self.assertEqual(d.placement_size, 100 * _GIB) + self.assertEqual(d.backend_size, 150 * _GIB) + self.assertEqual(d.unused_size(), 0) + self.assertEqual([r["unused_size"] for r in res["devices"]], [0, 0, 0]) self.assertEqual(h.rpc.mock_calls[:3], [ call.alceml_bdev_backend_resize_begin( - "alceml_dev-1", request_timeout=device_controller.RESIZE_RPC_TIMEOUT_SEC), - call.bdev_aio_rescan("aio_dev-1"), + "alceml_dev-0", request_timeout=device_controller.RESIZE_RPC_TIMEOUT_SEC), + call.bdev_aio_rescan("aio_dev-0"), call.alceml_bdev_backend_resize_end( - "alceml_dev-1", b_fail=False, cntr_path="", + "alceml_dev-0", b_fail=False, cntr_path="", request_timeout=device_controller.RESIZE_RPC_TIMEOUT_SEC), ]) - h.events.device_resized.assert_called_once() - - def test_second_growth_keeps_original_placement(self): - h = _ResizeHarness(_aio_dev(size_gib=200, placement_gib=100), 300 * _GIB) - h.run() - self.assertEqual(h.dev.placement_size, 100 * _GIB) - self.assertEqual(h.dev.size, 300 * _GIB) - - def test_uneven_growth_warns(self): - other = _aio_dev("dev-2", size_gib=100, order=1) - h = _ResizeHarness(_aio_dev(size_gib=100), 200 * _GIB, others=[other]) - res = h.run() - self.assertTrue(res["resized"]) - self.assertFalse(res["uniform"]) - self.assertIn("unevenly", res["warning"]) - - def test_even_growth_is_uniform(self): - other = _aio_dev("dev-2", size_gib=200, placement_gib=100, order=1) - h = _ResizeHarness(_aio_dev(size_gib=100), 200 * _GIB, others=[other]) - self.assertTrue(h.run()["uniform"]) - - def test_journal_device_is_ignored_for_uniformity(self): - jm = _aio_dev("jm", size_gib=10, status=NVMeDevice.STATUS_JM, order=1) - h = _ResizeHarness(_aio_dev(size_gib=100), 200 * _GIB, others=[jm]) - self.assertTrue(h.run()["uniform"]) - - def test_unchanged_size_is_noop(self): - h = _ResizeHarness(_aio_dev(size_gib=100), 100 * _GIB) - res = h.run() - self.assertFalse(res["resized"]) + self.assertEqual(h.rpc.alceml_bdev_backend_resize_begin.call_count, 3) + self.assertEqual(h.events.device_resized.call_count, 3) + + def test_mixed_factors_grow_all_by_the_smallest(self): + h = _GrowHarness([150, 150, 150, 200, 200, 200]) + res = h.grow() + self.assertTrue(res["grown"]) + self.assertAlmostEqual(res["factor"], 1.5) + self.assertEqual(h.sizes(), [150] * 6) + self.assertEqual([d.unused_size() // _GIB for d in h.devs], [0, 0, 0, 50, 50, 50]) + self.assertEqual([d.placement_size // _GIB for d in h.devs], [100] * 6) + # Every alceml reopens: all of them take their whole block device. + self.assertEqual(h.rpc.alceml_bdev_backend_resize_begin.call_count, 6) + + def test_weights_never_change(self): + h = _GrowHarness([150, 200]) + before = TestPlacementWeight()._build_map(_node([copy_dev(d) for d in h.devs])) + h.grow() + after = TestPlacementWeight()._build_map(h.node) + self.assertEqual(before["map_prob"], after["map_prob"]) + + def test_device_that_did_not_grow_stops_all_growth(self): + h = _GrowHarness([150, 100, 200]) + res = h.grow() + self.assertFalse(res["grown"]) + self.assertEqual(res["factor"], 1.0) + self.assertIn("Not every block device has grown", res["reason"]) + self.assertEqual(h.sizes(), [100, 100, 100]) h.rpc.alceml_bdev_backend_resize_begin.assert_not_called() + # The rescanned sizes are recorded all the same. + self.assertEqual([d.backend_size // _GIB for d in h.devs], [150, 100, 200]) - def test_shrink_is_refused(self): - h = _ResizeHarness(_aio_dev(size_gib=100), 50 * _GIB) - with self.assertRaisesRegex(ValueError, "shrank"): - h.run() + def test_shrunk_device_is_refused_and_alerted(self): + h = _GrowHarness([150, 80, 150]) + res = h.grow() + self.assertFalse(res["grown"]) + self.assertIn("dev-1", res["errors"]) + self.assertIn("shrank", res["errors"]["dev-1"]) + self.assertEqual(res["shrunk"], ["dev-1"]) h.rpc.alceml_bdev_backend_resize_begin.assert_not_called() - self.assertEqual(h.dev.size, 100 * _GIB) + self.assertEqual(h.sizes(), [100, 100, 100]) + self.assertTrue(h.devs[1].is_shrunk()) + h.events.device_shrink_detected.assert_called_once() + alerts = alerts_controller._device_alerts("cluster-1", [h.node]) + shrunk = [a for a in alerts if a["kind"] == alerts_controller.DEVICE_SHRUNK] + self.assertEqual([a["device_id"] for a in shrunk], ["dev-1"]) + + def test_restored_lun_resolves_the_shrink(self): + h = _GrowHarness([100, 80]) + h.grow() + h.set_sizes([100, 100]) + res = h.grow() + self.assertFalse(h.devs[1].is_shrunk()) + self.assertEqual(res["errors"], {}) + alerts = alerts_controller._device_alerts("cluster-1", [h.node]) + self.assertFalse([a for a in alerts if a["kind"] == alerts_controller.DEVICE_SHRUNK]) + + def test_repeated_runs_are_idempotent(self): + h = _GrowHarness([150, 200]) + h.grow() + sizes = h.sizes() + for _ in range(2): + res = h.grow() + self.assertFalse(res["grown"]) + self.assertEqual(h.sizes(), sizes) + h.rpc.alceml_bdev_backend_resize_begin.assert_not_called() + # A later, larger growth still applies on top. + h.set_sizes([300, 300]) + res = h.grow() + self.assertTrue(res["grown"]) + self.assertAlmostEqual(res["factor"], 2.0) + self.assertEqual(h.sizes(), [300, 300]) + self.assertEqual([d.placement_size // _GIB for d in h.devs], [100, 100]) + + def test_offline_device_stops_all_growth(self): + h = _GrowHarness([150, 150]) + h.devs[1].status = NVMeDevice.STATUS_UNAVAILABLE + res = h.grow() + self.assertFalse(res["grown"]) + self.assertIn("dev-1", res["errors"]) + h.client.rescan_block_device.assert_not_called() + self.assertEqual(h.sizes(), [100, 100]) + + def test_journal_and_new_devices_are_ignored(self): + jm = _aio_dev("jm", size_gib=10, status=NVMeDevice.STATUS_JM, order=5) + new = _aio_dev("new", size_gib=50, status=NVMeDevice.STATUS_NEW, order=-1) + h = _GrowHarness([150, 150], others=[jm, new]) + res = h.grow() + self.assertTrue(res["grown"]) + self.assertEqual(jm.size, 10 * _GIB) + self.assertEqual(new.size, 50 * _GIB) def test_refuses_nvme_cluster(self): - h = _ResizeHarness(_aio_dev(), 200 * _GIB, device_mode=constants.DEVICE_MODE_NVME) + h = _GrowHarness([150], device_mode="nvme") with self.assertRaisesRegex(ValueError, "lblk"): - h.run() - - def test_refuses_offline_device(self): - h = _ResizeHarness(_aio_dev(status=NVMeDevice.STATUS_UNAVAILABLE), 200 * _GIB) - with self.assertRaisesRegex(ValueError, "online"): - h.run() - - def test_refuses_offline_node(self): - h = _ResizeHarness(_aio_dev(), 200 * _GIB, node_status=StorageNode.STATUS_OFFLINE) - with self.assertRaisesRegex(ValueError, "Storage node must be online"): - h.run() - - def test_refuses_partition(self): - dev = _aio_dev() - dev.is_partition = True - h = _ResizeHarness(dev, 200 * _GIB) - with self.assertRaisesRegex(ValueError, "Partitions"): - h.run() - - def test_refuses_testing_bdev(self): - dev = _aio_dev() - dev.testing_bdev = "test_dev-1" - h = _ResizeHarness(dev, 200 * _GIB) - with self.assertRaisesRegex(ValueError, "testing or hang"): - h.run() - - def test_host_rescan_error(self): - h = _ResizeHarness(_aio_dev(), 200 * _GIB) - h.client.rescan_block_device.return_value = (None, "not found") - with self.assertRaisesRegex(ValueError, "Rescan"): - h.run() - h.rpc.alceml_bdev_backend_resize_begin.assert_not_called() + h.grow() - def test_aio_rescan_failure_reopens_and_leaves_device_unchanged(self): - h = _ResizeHarness(_aio_dev(size_gib=100), 200 * _GIB) - h.rpc.bdev_aio_rescan.side_effect = RPCRemoteError("rescan failed", -1) - with self.assertRaisesRegex(ValueError, "unchanged"): - h.run() - h.rpc.alceml_bdev_backend_resize_end.assert_called_once_with( - "alceml_dev-1", b_fail=False, cntr_path="", - request_timeout=device_controller.RESIZE_RPC_TIMEOUT_SEC) - self.assertEqual(h.dev.size, 100 * _GIB) - self.assertEqual(h.dev.placement_size, 0) - h.set_unavail.assert_not_called() + def test_offline_node_stops_all_growth(self): + h = _GrowHarness([150], node_status=StorageNode.STATUS_OFFLINE) + res = h.grow() + self.assertFalse(res["grown"]) + self.assertIn("Storage node must be online", res["errors"]["dev-0"]) - def test_reopen_failure_fails_io_and_sets_unavailable(self): - h = _ResizeHarness(_aio_dev(size_gib=100), 200 * _GIB) - h.rpc.alceml_bdev_backend_resize_end.side_effect = RPCRemoteError("reopen", -7) - with self.assertRaisesRegex(ValueError, "unavailable"): - h.run() - calls = h.rpc.alceml_bdev_backend_resize_end.call_args_list - self.assertEqual(len(calls), device_controller.RESIZE_END_ATTEMPTS + 1) - self.assertTrue(calls[-1].kwargs["b_fail"]) - h.set_unavail.assert_called_once() - self.assertEqual(h.dev.size, 100 * _GIB) + def test_partition_or_testing_bdev_stops_all_growth(self): + h = _GrowHarness([150, 150]) + h.devs[0].is_partition = True + h.devs[1].testing_bdev = "x" + res = h.grow() + self.assertFalse(res["grown"]) + self.assertEqual(set(res["errors"]), {"dev-0", "dev-1"}) - def test_begin_failure_reopens(self): - h = _ResizeHarness(_aio_dev(size_gib=100), 200 * _GIB) - h.rpc.alceml_bdev_backend_resize_begin.side_effect = RPCRemoteError("busy", -6) - with self.assertRaisesRegex(ValueError, "resize_begin failed.*unchanged"): - h.run() - h.rpc.bdev_aio_rescan.assert_not_called() - h.rpc.alceml_bdev_backend_resize_end.assert_called() + def test_rescan_error_stops_all_growth(self): + h = _GrowHarness([150, 150]) + h.client.rescan_block_device.side_effect = lambda path: (None, "no such device") + res = h.grow() + self.assertFalse(res["grown"]) + h.rpc.alceml_bdev_backend_resize_begin.assert_not_called() + def test_reopen_failure_keeps_all_sizes_and_sets_unavailable(self): + h = _GrowHarness([150, 150]) + h.rpc.alceml_bdev_backend_resize_end.side_effect = [ + None, RPCRemoteError("reopen", -7), RPCRemoteError("reopen", -7), + RPCRemoteError("reopen", -7), None] + res = h.grow() + self.assertFalse(res["grown"]) + self.assertIn("dev-1", res["errors"]) + self.assertEqual(h.sizes(), [100, 100]) + h.set_unavail.assert_called_once() + h.events.device_resized.assert_not_called() -class TestGrowClusterDevices(unittest.TestCase): + def test_aio_rescan_failure_keeps_all_sizes(self): + h = _GrowHarness([150, 150]) + h.rpc.bdev_aio_rescan.side_effect = RPCRemoteError("rescan", -5) + res = h.grow() + self.assertFalse(res["grown"]) + self.assertIn("unchanged", res["errors"]["dev-0"]) + self.assertEqual(h.sizes(), [100, 100]) + h.set_unavail.assert_not_called() - def test_grows_all_aio_devices_and_collects_errors(self): - good = _aio_dev("dev-1", size_gib=100) - bad = _aio_dev("dev-2", size_gib=100, order=1) - bad.testing_bdev = "x" - jm = _aio_dev("jm", size_gib=10, status=NVMeDevice.STATUS_JM, order=2) - h = _ResizeHarness(good, 200 * _GIB, others=[bad, jm]) - res = h.run(device_controller.grow_cluster_devices, "cluster-1") - self.assertEqual([r["device_id"] for r in res["devices"]], ["dev-1"]) - self.assertIn("dev-2", res["errors"]) - self.assertNotIn("jm", res["errors"]) - - -class _GrowHarness(_ResizeHarness): - """Several devices whose block devices report their own new sizes.""" - - def __init__(self, sizes_gib, **kw): - devs = [_aio_dev(f"dev-{i}", size_gib=100, order=i) for i in range(len(sizes_gib))] - super().__init__(devs[0], 0, others=devs[1:], **kw) - self.devs = devs - by_path = {d.by_id_path: g * _GIB for d, g in zip(devs, sizes_gib)} - self.client.rescan_block_device.side_effect = lambda path: ( - {"device": "sdx", "size": by_path[path], "rescanned": []}, None) + def test_begin_failure_reopens(self): + h = _GrowHarness([150]) + h.rpc.alceml_bdev_backend_resize_begin.side_effect = RPCRemoteError("timeout", -1) + res = h.grow() + self.assertFalse(res["grown"]) + h.rpc.alceml_bdev_backend_resize_end.assert_called_with( + "alceml_dev-0", b_fail=False, cntr_path="", + request_timeout=device_controller.RESIZE_RPC_TIMEOUT_SEC) + h.set_unavail.assert_not_called() - def grow(self, **kw): - return self.run(lambda cid: device_controller.grow_cluster_devices(cid, **kw), "cluster-1") +class TestGrowAfterTakeover(unittest.TestCase): + """A multi-attach LUN another node took over is grown where its alceml + runs: the serving node, over the base bdev and path the takeover + recorded. The owner (back as a standby) is not touched.""" + + def _harness(self, host_status=StorageNode.STATUS_ONLINE): + h = _GrowHarness([150, 150]) + moved = h.devs[1] + moved.multi_attach = True + moved.serving_node_id = "node-2" + moved.node_paths = {"node-2": {"path": "/dev/disk/by-id/lun-on-2", "bdev": "aio_on_2"}} + host = _node([], status=host_status) + host.get_id.return_value = "node-2" + host.rpc = MagicMock() + host.rpc_client.return_value = host.rpc + host.agent = MagicMock() + host.agent.rescan_block_device.return_value = ({"size": 150 * _GIB}, None) + host.client.return_value = host.agent + h.db.get_storage_node_by_id.side_effect = lambda nid: host if nid == "node-2" else h.node + return h, moved, host + + def test_grown_on_the_serving_node(self): + h, moved, host = self._harness() + with patch.object(device_controller.utils, "lblk_io_engine_of_device", + return_value=constants.LBLK_IO_ENGINE_AIO): + res = h.grow() + self.assertTrue(res["grown"], res) + self.assertEqual(h.sizes(), [150, 150]) + host.agent.rescan_block_device.assert_called_once_with("/dev/disk/by-id/lun-on-2") + host.rpc.alceml_bdev_backend_resize_begin.assert_called_once_with( + "alceml_dev-1", request_timeout=device_controller.RESIZE_RPC_TIMEOUT_SEC) + host.rpc.bdev_aio_rescan.assert_called_once_with("aio_on_2") + owner_calls = [c for c in h.rpc.mock_calls if "alceml_dev-1" in str(c) or "aio_dev-1" in str(c)] + self.assertEqual(owner_calls, []) + self.assertNotIn(moved.by_id_path, [c.args[0] for c in h.client.rescan_block_device.call_args_list]) + + def test_serving_node_offline_grows_nothing(self): + h, _, host = self._harness(host_status=StorageNode.STATUS_OFFLINE) + res = h.grow() + self.assertFalse(res["grown"]) + self.assertIn("served by node node-2", res["errors"]["dev-1"]) + self.assertEqual(h.sizes(), [100, 100]) + host.rpc.alceml_bdev_backend_resize_begin.assert_not_called() + h.rpc.alceml_bdev_backend_resize_begin.assert_not_called() -class TestRebalanceAfterGrowth(unittest.TestCase): - def test_uniform_growth_keeps_weights_and_does_not_rebalance(self): - h = _GrowHarness([200, 200, 200]) - res = h.grow() - self.assertTrue(res["uniform"]) - self.assertIsNone(res["rebalance"]) - self.assertEqual([d.weight_size() for d in h.devs], [100 * _GIB] * 3) - h.push.assert_not_called() - h.rebalance_task.assert_not_called() - - def test_uneven_growth_reweights_pushes_and_rebalances(self): - h = _GrowHarness([200, 300, 100]) - res = h.grow() - self.assertFalse(res["uniform"]) - self.assertEqual(res["rebalance"], {"weights_updated": True, "started": True, "reason": ""}) - self.assertEqual([d.weight_size() for d in h.devs], [200 * _GIB, 300 * _GIB, 100 * _GIB]) - h.push.assert_called_once_with(h.node) - h.rebalance_task.assert_called_once_with("node-1") - self.assertFalse(h.cluster.reweight_pending) - h.events.devices_reweighted.assert_called_once_with("cluster-1", True) - - def test_no_rebalance_keeps_frozen_weights(self): - h = _GrowHarness([200, 300]) - res = h.grow(rebalance=False) - self.assertFalse(res["uniform"]) - self.assertIsNone(res["rebalance"]) - self.assertEqual([d.weight_size() for d in h.devs], [100 * _GIB] * 2) - h.push.assert_not_called() - - def test_open_migration_blocks_reweight(self): - h = _GrowHarness([200, 300]) - task = MagicMock(function_name=device_controller.JobSchedule.FN_NEW_DEV_MIG, - status="running", canceled=False, uuid="t-1") - h.db.get_job_tasks.return_value = [task] - res = h.grow() - self.assertFalse(res["rebalance"]["started"]) - self.assertIn("t-1", res["rebalance"]["reason"]) - self.assertEqual([d.weight_size() for d in h.devs], [100 * _GIB] * 2) - self.assertFalse(h.cluster.reweight_pending) - h.push.assert_not_called() - - def test_inactive_cluster_blocks_reweight(self): - h = _GrowHarness([200, 300]) - h.cluster.status = "degraded" - res = h.grow() - self.assertIn("degraded", res["rebalance"]["reason"]) - self.assertEqual([d.weight_size() for d in h.devs], [100 * _GIB] * 2) +class TestUnusedCapacity(unittest.TestCase): + """The capacity statistics leave out the part of a block device beyond + the device size: alceml spans the whole block device.""" + + def test_device_stats_cap_total_at_device_size(self): + from simplyblock_core.models.stats import DeviceStatObject + from simplyblock_core.services import capacity_and_stats_collector as collector + dev = _aio_dev(size_gib=150, placement_gib=100) + dev.backend_size = 200 * _GIB + cl = MagicMock() + cl.get_id.return_value = "cluster-1" + cl.distr_ndcs, cl.distr_npcs = 1, 1 + page = 2 * 1024 * 1024 + cap = {"res": 1, "npages_nmax": (200 * _GIB) // page, "npages_used": 0, "pba_page_size": page} + db = MagicMock() + db.get_device_stats.return_value = [] + with patch.object(collector, "db", db), patch.object(collector, "last_object_record", {}), \ + patch.object(DeviceStatObject, "write_to_db", lambda self, kv: True): + rec = collector.add_device_stats(cl, dev, cap, None) + self.assertEqual(rec.size_total_raw, 150 * _GIB) - def test_failed_growth_does_not_reweight(self): - h = _GrowHarness([200, 300]) - h.devs[1].testing_bdev = "x" - res = h.grow() - self.assertIn("dev-1", res["errors"]) - self.assertFalse(res["rebalance"]["started"]) - self.assertIn("failed to grow", res["rebalance"]["reason"]) - h.push.assert_not_called() + def test_alert_free_device_reports_unused(self): + dev = _aio_dev(size_gib=150, placement_gib=100) + dev.backend_size = 200 * _GIB + self.assertEqual(dev.unused_size(), 50 * _GIB) + self.assertFalse(dev.is_shrunk()) - def test_failed_push_keeps_pending_and_retry_finishes(self): - h = _GrowHarness([200, 300]) - h.push_ok = False - res = h.grow() - self.assertFalse(res["rebalance"]["started"]) - self.assertIn("push failed", res["rebalance"]["reason"]) - self.assertTrue(h.cluster.reweight_pending) - self.assertEqual([d.weight_size() for d in h.devs], [200 * _GIB, 300 * _GIB]) - h.rebalance_task.assert_not_called() - - # Re-run: sizes unchanged, growth now looks uniform, but the pending - # flag makes it push again and start the rebalance. - h.push_ok = True - res = h.grow() - self.assertTrue(res["rebalance"]["started"]) - self.assertFalse(h.cluster.reweight_pending) - h.rebalance_task.assert_called_once_with("node-1") - - def test_weights_updated_without_volume_stores_to_rebalance(self): - h = _GrowHarness([200, 300]) - - def _grow_without_queue(cluster_id): - h.rebalance_task.return_value = False - return device_controller.grow_cluster_devices(cluster_id) - res = h.run(_grow_without_queue, "cluster-1") - self.assertTrue(res["rebalance"]["weights_updated"]) - self.assertFalse(res["rebalance"]["started"]) - self.assertIn("no rebalance was queued", res["rebalance"]["reason"]) - self.assertFalse(h.cluster.reweight_pending) - self.assertEqual([d.weight_size() for d in h.devs], [200 * _GIB, 300 * _GIB]) - - def test_cluster_map_carries_new_weights_after_reweight(self): - h = _GrowHarness([200, 300]) - h.grow() - cl_map = TestPlacementWeight._build_map(None, h.node) - weights = [i["weight"] for i in cl_map["map_prob"][0]["items"]] - self.assertEqual(weights, [200, 300]) + +def copy_dev(d): + return NVMeDevice(d.to_dict()) class TestScaledPlacementSize(unittest.TestCase): @@ -413,7 +411,7 @@ def test_uniform_growth_scales_new_device(self): _aio_dev("b", size_gib=200, placement_gib=100, order=1)]) self.assertEqual(storage_node_ops.scaled_placement_size(db, "cluster-1", 200 * _GIB), 100 * _GIB) - def test_uneven_growth_uses_full_size(self): + def test_inconsistent_ratios_use_full_size(self): db = self._db([_aio_dev("a", size_gib=200, placement_gib=100), _aio_dev("b", size_gib=100, order=1)]) self.assertEqual(storage_node_ops.scaled_placement_size(db, "cluster-1", 200 * _GIB), 0) @@ -496,3 +494,36 @@ def test_virtio_without_rescan_reads_size(self): if __name__ == "__main__": unittest.main() + + +class TestStatsSourceAfterTakeover(unittest.TestCase): + """Capacity and I/O stats of a taken-over LUN come from the node whose + alceml serves it; read on the owner they were missing, and the cluster + capacity lost one device per takeover.""" + + def setUp(self): + from simplyblock_core.services import capacity_and_stats_collector as col + self.col = col + self.owner = _node([]) + self.owner_rpc = MagicMock() + self.host = _node([]) + self.host.get_id.return_value = "node-2" + self.host_rpc = MagicMock() + self.host.rpc_client.return_value = self.host_rpc + self.nodes = {"node-1": self.owner, "node-2": self.host} + self.dev = _aio_dev("dev-1") + + def test_owner_serves(self): + self.assertEqual(self.col.device_stats_source(self.dev, self.owner, self.owner_rpc, self.nodes), + (self.owner_rpc, "aio_dev-1")) + + def test_taken_over_reads_the_serving_node(self): + self.dev.multi_attach, self.dev.serving_node_id = True, "node-2" + self.dev.node_paths = {"node-2": {"path": "/dev/x", "bdev": "aio_on_2"}} + self.assertEqual(self.col.device_stats_source(self.dev, self.owner, self.owner_rpc, self.nodes), + (self.host_rpc, "aio_on_2")) + + def test_serving_node_offline_is_skipped(self): + self.dev.multi_attach, self.dev.serving_node_id = True, "node-2" + self.host.status = StorageNode.STATUS_OFFLINE + self.assertIsNone(self.col.device_stats_source(self.dev, self.owner, self.owner_rpc, self.nodes)) diff --git a/tests/unit/test_lblk_eligibility.py b/tests/unit/test_lblk_eligibility.py index c5d04a6759..ee04d55f92 100644 --- a/tests/unit/test_lblk_eligibility.py +++ b/tests/unit/test_lblk_eligibility.py @@ -272,6 +272,30 @@ def test_name_fallback_when_serial_unknown(self): names = {e["name"] for e in resolved} self.assertEqual(names, {"sdb", "sdc"}) + def test_no_name_fallback_for_array_luns(self): + # Multi-attach: S1 is a LUN the node is fenced from; after a reboot + # its old name sdb is another disk (FTT1 bed: the journal volume). + host = [_blk("sdb", serial="JOURNAL"), _blk("sdc", serial="S2")] + resolved, missing = utils.resolve_lblk_entries(self.CONFIGURED, host, + name_fallback=False) + self.assertEqual([e["serial"] for e in resolved], ["S2"]) + self.assertEqual([e["serial"] for e in missing], ["S1"]) + + def test_serial_less_entry_still_falls_back_by_name(self): + configured = [{"name": "sdb", "serial": "", "by_id": "", "size": 100, "numa": 0}] + resolved, missing = utils.resolve_lblk_entries(configured, [_blk("sdb", serial="X")], + name_fallback=False) + self.assertEqual((len(resolved), missing), (1, [])) + + def test_node_paths_turn_name_fallback_off_on_multi_attach(self): + from simplyblock_core import storage_node_ops + from tests._source import code_of + for fn in (storage_node_ops.add_node, storage_node_ops._restart_storage_node_impl): + code = code_of(fn) + self.assertIn("resolve_lblk_entries(", code, fn.__name__) + self.assertIn("name_fallback=not multi_attach.is_multi_attach(cluster)", code, + fn.__name__) + def test_missing_device_reported(self): host = [_blk("sdc", serial="S2")] resolved, missing = utils.resolve_lblk_entries(self.CONFIGURED, host) @@ -356,6 +380,12 @@ def test_single_lblk_entry_rejected(self): node = self._node(lblk_devices=[{"name": "sdb", "serial": "S1", "size": 100}]) self.assertFalse(utils.validate_node_config(node)) + def test_single_journal_only_entry_valid(self): + # sn configure --lblk-journal-only: the data device is the array LUN + # added at add-node, which checks the minimum again with it counted. + node = self._node(lblk_devices=[{"name": "sdb", "serial": "S1", "size": 100, "journal": True}]) + self.assertTrue(utils.validate_node_config(node)) + def test_two_journal_flags_rejected(self): node = self._node(lblk_devices=[ {"name": "sdb1", "serial": "S1", "size": 100, "journal": True}, diff --git a/tests/unit/test_lblk_io_engine.py b/tests/unit/test_lblk_io_engine.py new file mode 100644 index 0000000000..ef9a1f7050 --- /dev/null +++ b/tests/unit/test_lblk_io_engine.py @@ -0,0 +1,477 @@ +"""Unit tests for the I/O engine of lblk base bdevs. + +SPDK's bdev_aio submits with RWF_NOWAIT, which device-mapper honours only +when every target does; dm-multipath does not, so a multipath map cannot be +served by bdev_aio at all. Device-mapper nodes are served by bdev_uring +instead, everything else keeps bdev_aio. The bdev name (aio_) and +NVMeDevice.bdev_type "aio" are the same for both engines. + +Covered: + - utils.lblk_io_engine_for: dm-* kernel name, lsblk TYPE mpath/lvm/crypt/dm, + a kpartx partition of a multipath map, /dev/mapper and by-id dm-* paths + (older agent), sd/nvme disks and partitions and md arrays -> aio. + - utils.resolve_lblk_entries carries kname/type, the derived engine and the + recorded engine (same device only). + - utils.addAioDevices: dm device -> bdev_uring_create with the block size, + recorded on the device and the node's lblk_devices entry; sd/md -> aio; + recorded uring kept; recorded aio on a dm node -> uring; older agent; + an existing bdev's engine is kept. + - delete / rescan route by engine; device growth rescans a uring device + with bdev_uring_rescan. + - storage_node_ops._reconcile_discovered_devices takes the engine over. + - device_controller.restart_device / new_device_from_failed create the + bdev with the device's engine. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import constants, utils +from simplyblock_core.controllers import device_controller +from simplyblock_core.models.nvme_device import JMDevice, NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.storage_node_ops import _reconcile_discovered_devices + +GIB = 1 << 30 +AIO = constants.LBLK_IO_ENGINE_AIO +URING = constants.LBLK_IO_ENGINE_URING + + +def _entry(serial="S1", name="sdb", current_path="/dev/sdb", by_id=None, **extra): + e = {"name": name, "serial": serial, + "by_id": by_id if by_id is not None else f"/dev/disk/by-id/wwn-{serial}", + "size": 100 * GIB, "numa": 0, "model": "M", "current_path": current_path, + "has_partitions": False, "logical_block_size": 512, + "physical_block_size": 4096} + e.update(extra) + return e + + +def _mpath_entry(serial="S1", **extra): + return _entry(serial=serial, name="mpatha", current_path="/dev/mpatha", + by_id=f"/dev/disk/by-id/dm-uuid-mpath-{serial}", + kname="dm-0", dev_type="mpath", **extra) + + +def _snode(devices=(), lblk_devices=None, jm_device=None): + n = StorageNode() + n.uuid = "node-1" + n.cluster_id = "cluster-1" + n.nvme_devices = list(devices) + n.lblk_devices = lblk_devices if lblk_devices is not None else [ + {"name": "sdb", "serial": "S1", "size": 100 * GIB}] + if jm_device is not None: + n.jm_device = jm_device + return n + + +def _record(serial="S1", engine=AIO, aio_block_size=4096, + status=NVMeDevice.STATUS_ONLINE): + d = NVMeDevice() + d.uuid = f"dev-{serial}" + d.serial_number = serial + d.bdev_type = "aio" + d.nvme_bdev = utils.aio_bdev_name_for_serial(serial) + d.aio_block_size = aio_block_size + d.lblk_io_engine = engine + d.status = status + return d + + +class _FakeRpc: + """SPDK fake with both base-bdev engines; records every create.""" + + def __init__(self, existing=None): + self.host = "test-host" + self.bdevs = dict(existing or {}) + self.creates = [] # (engine, name, filename, block_size) + + def bdev_get(self, name): + return self.bdevs.get(name) + + def _create(self, engine, name, filename, block_size): + self.creates.append((engine, name, filename, block_size)) + bs = block_size or 512 + self.bdevs[name] = {"name": name, "block_size": bs, + "num_blocks": 100 * GIB // bs, + "driver_specific": {engine: {"filename": filename}}} + return name + + def bdev_aio_create(self, name, filename, block_size=0): + return self._create(AIO, name, filename, block_size) + + def bdev_uring_create(self, name, filename, block_size=0): + return self._create(URING, name, filename, block_size) + + def bdev_examine(self, name): + return True + + def bdev_wait_for_examine(self): + return True + + def bdev_set_qd_sampling_period(self, name, period): + return True + + +class TestEngineRule(unittest.TestCase): + + def test_dm_kernel_name_is_uring(self): + self.assertEqual(utils.lblk_io_engine_for({"name": "mpatha", "kname": "dm-3"}), URING) + + def test_dm_types_are_uring(self): + for t in ("mpath", "lvm", "crypt", "dm"): + with self.subTest(t=t): + self.assertEqual(utils.lblk_io_engine_for({"name": "x", "type": t}), URING) + + def test_kpartx_partition_of_multipath_is_uring(self): + self.assertEqual(utils.lblk_io_engine_for( + {"name": "mpatha1", "kname": "dm-4", "type": "part"}), URING) + + def test_mapper_and_dm_paths_are_uring(self): + for path in ("/dev/mapper/mpatha", "/dev/dm-0", + "/dev/disk/by-id/dm-uuid-mpath-3600a", "/dev/disk/by-id/dm-name-mpatha"): + with self.subTest(path=path): + self.assertEqual(utils.lblk_io_engine_for( + {"name": "x", "device_path": path}), URING) + + def test_plain_devices_are_aio(self): + for dev in ({"name": "sdb", "kname": "sdb", "type": "disk", + "device_path": "/dev/sdb", "by_id_path": "/dev/disk/by-id/wwn-0x5"}, + {"name": "nvme0n1p2", "kname": "nvme0n1p2", "type": "part"}, + {"name": "sdc1", "type": "part", "device_path": "/dev/sdc1"}, + {"name": "md0", "kname": "md0", "type": "raid5"}, + {"name": "md127", "type": "raid1", "device_path": "/dev/md127"}): + with self.subTest(dev=dev["name"]): + self.assertEqual(utils.lblk_io_engine_for(dev), AIO) + + def test_older_agent_without_fields_is_aio(self): + self.assertEqual(utils.lblk_io_engine_for( + {"name": "sdb", "device_path": "/dev/sdb", "by_id_path": ""}), AIO) + + def test_choose_prefers_recorded_unless_rule_says_uring(self): + self.assertEqual(utils.choose_lblk_io_engine(None, AIO), AIO) + self.assertEqual(utils.choose_lblk_io_engine(URING, AIO), URING) + self.assertEqual(utils.choose_lblk_io_engine(AIO, URING), URING) + self.assertEqual(utils.choose_lblk_io_engine(AIO, AIO), AIO) + + +class TestResolveCarriesEngine(unittest.TestCase): + + def _live(self, serial, **extra): + d = {"name": "mpatha", "kname": "dm-0", "type": "mpath", + "device_path": "/dev/mpatha", "serial": serial, "by_id_path": "", + "size": 100 * GIB, "numa_node": 0} + d.update(extra) + return d + + def test_dm_device_resolves_to_uring(self): + resolved, _ = utils.resolve_lblk_entries( + [{"name": "mpatha", "serial": "S1", "size": 1}], [self._live("S1")]) + self.assertEqual(resolved[0]["kname"], "dm-0") + self.assertEqual(resolved[0]["dev_type"], "mpath") + self.assertEqual(resolved[0]["io_engine"], URING) + + def test_disk_resolves_to_aio(self): + live = self._live("S1", name="sdb", kname="sdb", type="disk", device_path="/dev/sdb") + resolved, _ = utils.resolve_lblk_entries( + [{"name": "sdb", "serial": "S1", "size": 1}], [live]) + self.assertEqual(resolved[0]["io_engine"], AIO) + + def test_older_agent_mapper_path(self): + live = {"name": "mpatha", "device_path": "/dev/mapper/mpatha", "serial": "S1", + "by_id_path": "", "size": 100 * GIB, "numa_node": 0} + resolved, _ = utils.resolve_lblk_entries( + [{"name": "mpatha", "serial": "S1", "size": 1}], [live]) + self.assertEqual(resolved[0]["io_engine"], URING) + + def test_recorded_engine_carried_for_same_device_only(self): + live = self._live("S1", name="sdb", kname="sdb", type="disk", device_path="/dev/sdb") + resolved, _ = utils.resolve_lblk_entries( + [{"name": "sdb", "serial": "S1", "size": 1, "lblk_io_engine": URING}], [live]) + self.assertEqual(resolved[0]["lblk_io_engine"], URING) + live = dict(live, serial="OTHER") + resolved, _ = utils.resolve_lblk_entries( + [{"name": "sdb", "serial": "S1", "size": 1, "lblk_io_engine": URING}], [live]) + self.assertNotIn("lblk_io_engine", resolved[0]) + + +class TestAddAioDevicesEngine(unittest.TestCase): + + def test_new_multipath_device_created_with_uring(self): + rpc = _FakeRpc() + snode = _snode() + devs = utils.addAioDevices(rpc, snode, [_mpath_entry(io_engine=URING)]) + self.assertEqual(rpc.creates, [(URING, "aio_S1", "/dev/disk/by-id/dm-uuid-mpath-S1", 4096)]) + self.assertEqual(devs[0].lblk_io_engine, URING) + self.assertEqual(devs[0].nvme_bdev, "aio_S1") + self.assertEqual(devs[0].bdev_type, "aio") + self.assertEqual(devs[0].aio_block_size, 4096) + self.assertEqual(snode.lblk_devices[0]["lblk_io_engine"], URING) + + def test_new_disk_created_with_aio(self): + rpc = _FakeRpc() + snode = _snode() + devs = utils.addAioDevices(rpc, snode, [_entry(io_engine=AIO)]) + self.assertEqual(rpc.creates[0][0], AIO) + self.assertEqual(devs[0].lblk_io_engine, AIO) + self.assertEqual(snode.lblk_devices[0]["lblk_io_engine"], AIO) + + def test_md_array_created_with_aio(self): + rpc = _FakeRpc() + entry = _entry(name="md0", current_path="/dev/md0", by_id="/dev/disk/by-id/md-uuid-1", + kname="md0", dev_type="raid5") + devs = utils.addAioDevices(rpc, _snode(), [entry]) + self.assertEqual(rpc.creates[0][0], AIO) + self.assertEqual(devs[0].lblk_io_engine, AIO) + + def test_entry_without_derived_engine_uses_rule(self): + rpc = _FakeRpc() + entry = _mpath_entry() + utils.addAioDevices(rpc, _snode(), [entry]) + self.assertEqual(rpc.creates[0][0], URING) + + def test_older_agent_disk_without_fields_is_aio(self): + rpc = _FakeRpc() + entry = _entry() + entry.pop("logical_block_size") + utils.addAioDevices(rpc, _snode(), [entry]) + self.assertEqual(rpc.creates[0][0], AIO) + + def test_recorded_uring_kept(self): + # e.g. an older agent that reports the map under a by-id wwn link only + rpc = _FakeRpc() + snode = _snode(devices=[_record(engine=URING)]) + devs = utils.addAioDevices(rpc, snode, [_entry(io_engine=AIO)]) + self.assertEqual(rpc.creates, [(URING, "aio_S1", "/dev/disk/by-id/wwn-S1", 4096)]) + self.assertEqual(devs[0].lblk_io_engine, URING) + + def test_recorded_uring_on_lblk_entry_kept(self): + rpc = _FakeRpc() + snode = _snode(devices=[_record(engine=AIO)], + lblk_devices=[{"name": "sdb", "serial": "S1", "lblk_io_engine": URING}]) + utils.addAioDevices(rpc, snode, [_entry()]) + self.assertEqual(rpc.creates[0][0], URING) + + def test_recorded_aio_on_dm_node_switches_to_uring(self): + rpc = _FakeRpc() + snode = _snode(devices=[_record(engine=AIO, aio_block_size=512)]) + with self.assertLogs(utils.logger, level="WARNING") as logs: + devs = utils.addAioDevices(rpc, snode, [_mpath_entry(io_engine=URING)]) + self.assertEqual(rpc.creates[0][0], URING) + self.assertEqual(rpc.creates[0][3], 512) # the recorded block size is kept + self.assertEqual(devs[0].lblk_io_engine, URING) + self.assertTrue(any("using uring" in m for m in logs.output)) + + def test_journal_record_engine_used(self): + jm = JMDevice() + jm.serial_number = "S1" + jm.device_data_dict = {"serial_number": "S1", "lblk_io_engine": URING, + "aio_block_size": 4096} + rpc = _FakeRpc() + utils.addAioDevices(rpc, _snode(jm_device=jm), [_entry()]) + self.assertEqual(rpc.creates[0][0], URING) + + def test_existing_bdev_engine_recorded(self): + existing = {"aio_S1": {"name": "aio_S1", "block_size": 4096, + "num_blocks": 100 * GIB // 4096, + "driver_specific": {URING: {"filename": "/dev/dm-0"}}}} + rpc = _FakeRpc(existing=existing) + devs = utils.addAioDevices(rpc, _snode(devices=[_record(engine=URING)]), [_entry()]) + self.assertEqual(rpc.creates, []) + self.assertEqual(devs[0].lblk_io_engine, URING) + + +class TestDeleteRescanRouting(unittest.TestCase): + + def test_delete_routes_by_engine(self): + rpc = MagicMock() + utils.delete_lblk_bdev(rpc, "aio_S1", URING) + rpc.bdev_uring_delete.assert_called_once_with("aio_S1") + rpc.bdev_aio_delete.assert_not_called() + rpc = MagicMock() + utils.delete_lblk_bdev(rpc, "aio_S1", AIO) + rpc.bdev_aio_delete.assert_called_once_with("aio_S1") + rpc.bdev_uring_delete.assert_not_called() + + def test_rescan_routes_by_engine(self): + rpc = MagicMock() + utils.rescan_lblk_bdev(rpc, "aio_S1", URING) + rpc.bdev_uring_rescan.assert_called_once_with("aio_S1") + rpc.bdev_aio_rescan.assert_not_called() + rpc = MagicMock() + utils.rescan_lblk_bdev(rpc, "aio_S1") + rpc.bdev_aio_rescan.assert_called_once_with("aio_S1") + + def test_create_routes_by_engine_with_block_size(self): + rpc = _FakeRpc() + self.assertEqual(utils.create_aio_bdev(rpc, "aio_S1", "/dev/dm-0", 4096, + io_engine=URING), 4096) + self.assertEqual(rpc.creates, [(URING, "aio_S1", "/dev/dm-0", 4096)]) + + def test_reopen_alceml_rescans_uring_device(self): + dev = _record(engine=URING) + dev.alceml_bdev = "alceml_1" + snode = _snode(devices=[dev]) + rpc = MagicMock() + with patch.object(StorageNode, "rpc_client", return_value=rpc): + device_controller._reopen_alceml(snode, dev) + rpc.bdev_uring_rescan.assert_called_once_with("aio_S1") + rpc.bdev_aio_rescan.assert_not_called() + + def test_reopen_alceml_rescans_aio_device(self): + dev = _record(engine=AIO) + dev.alceml_bdev = "alceml_1" + snode = _snode(devices=[dev]) + rpc = MagicMock() + with patch.object(StorageNode, "rpc_client", return_value=rpc): + device_controller._reopen_alceml(snode, dev) + rpc.bdev_aio_rescan.assert_called_once_with("aio_S1") + rpc.bdev_uring_rescan.assert_not_called() + + +class TestReconcileTakesEngine(unittest.TestCase): + + def setUp(self): + # Not a multi-attach cluster: reconcile's foreign-LUN lookup (a DB + # read, multi-attach only) finds nothing. + p = patch("simplyblock_core.storage_node_ops._foreign_lun_keys", return_value=set()) + p.start() + self.addCleanup(p.stop) + + def test_engine_taken_over(self): + db_dev = _record(engine=AIO) + found = _record(engine=URING) + _reconcile_discovered_devices(_snode(devices=[db_dev]), [found]) + self.assertEqual(db_dev.lblk_io_engine, URING) + + +class TestDeviceControllerEngine(unittest.TestCase): + + def _restart(self, device, inventory=(), lblk_devices=None): + device.node_id = "node-1" + device.cluster_id = "cluster-1" + device.nvmf_nqn = "" + device.alceml_bdev = "" + snode = MagicMock() + snode.cluster_id = "cluster-1" + snode.nvme_devices = [device] + snode.jm_device = None + snode.lblk_devices = lblk_devices or [] + + rpc = MagicMock() + state = {"present": False} + rpc.bdev_get.side_effect = ( + lambda name: {"name": name, "block_size": 4096} if state["present"] else None) + + def _create(name, filename, block_size=0): + state["present"] = True + return name + + rpc.bdev_aio_create.side_effect = _create + rpc.bdev_uring_create.side_effect = _create + snode.rpc_client.return_value = rpc + snode.client.return_value.get_blockdevices.return_value = (list(inventory), None) + + db = MagicMock() + db.get_storage_device_by_id.return_value = device + db.get_storage_node_by_id.return_value = snode + with patch.object(device_controller, "DBController", return_value=db), \ + patch.object(device_controller.tasks_controller, + "get_active_dev_restart_task", return_value=None), \ + patch.object(device_controller, "device_set_retries_exhausted"), \ + patch.object(device_controller, "device_set_unavailable"), \ + patch.object(device_controller, "_def_create_device_stack", return_value=True), \ + patch.object(device_controller, "device_set_io_error"), \ + patch.object(device_controller, "device_set_online"), \ + patch.object(device_controller, "device_events"): + result = device_controller.restart_device(device.uuid) + return result, rpc + + def test_restart_uring_device(self): + device = _record(engine=URING, status=NVMeDevice.STATUS_REMOVED) + device.by_id_path = "/dev/disk/by-id/wwn-1" + result, rpc = self._restart(device) + self.assertTrue(result) + rpc.bdev_uring_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-1", block_size=4096) + rpc.bdev_aio_create.assert_not_called() + + def test_restart_aio_device(self): + device = _record(engine=AIO, status=NVMeDevice.STATUS_REMOVED) + device.by_id_path = "/dev/disk/by-id/wwn-1" + result, rpc = self._restart(device) + self.assertTrue(result) + rpc.bdev_aio_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-1", block_size=4096) + rpc.bdev_uring_create.assert_not_called() + + def test_restart_recorded_aio_dm_device_gets_uring(self): + device = _record(engine=AIO, status=NVMeDevice.STATUS_REMOVED) + device.by_id_path = "/dev/disk/by-id/wwn-1" + inventory = [{"serial": "S1", "name": "mpatha", "kname": "dm-0", "type": "mpath", + "device_path": "/dev/mpatha", + "by_id_path": "/dev/disk/by-id/dm-uuid-mpath-S1"}] + result, rpc = self._restart(device, inventory=inventory) + self.assertTrue(result) + rpc.bdev_uring_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/dm-uuid-mpath-S1", block_size=4096) + self.assertEqual(device.lblk_io_engine, URING) + + def test_restart_engine_from_lblk_entry(self): + device = _record(engine=AIO, status=NVMeDevice.STATUS_REMOVED) + device.by_id_path = "/dev/disk/by-id/wwn-1" + result, rpc = self._restart( + device, lblk_devices=[{"serial": "S1", "lblk_io_engine": URING}]) + self.assertTrue(result) + rpc.bdev_uring_create.assert_called_once() + + def test_new_device_from_failed_keeps_engine(self): + device = _record(engine=URING, status=NVMeDevice.STATUS_FAILED_AND_MIGRATED) + device.by_id_path = "/dev/disk/by-id/wwn-1" + node = MagicMock() + node.get_id.return_value = "node-1" + node.nvme_devices = [device] + node.lblk_devices = [] + rpc = MagicMock() + state = {"present": False} + rpc.bdev_get.side_effect = ( + lambda name: {"name": name, "block_size": 4096} if state["present"] else None) + + def _create(name, filename, block_size=0): + state["present"] = True + return name + + rpc.bdev_uring_create.side_effect = _create + node.rpc_client.return_value = rpc + node.client.return_value.get_blockdevices.return_value = ([], None) + db = MagicMock() + db.get_storage_node_by_device_id.return_value = node + with patch.object(device_controller, "DBController", return_value=db): + self.assertTrue(device_controller.new_device_from_failed(device.uuid)) + rpc.bdev_uring_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-1", block_size=4096) + rpc.bdev_aio_create.assert_not_called() + + +class TestNoDirectAioRpcCalls(unittest.TestCase): + """Every bdev_aio_create/delete/rescan goes through the engine-aware + helpers in utils.""" + + def test_no_direct_calls_outside_helpers(self): + import pathlib + import re + root = pathlib.Path(utils.__file__).resolve().parents[2] + pattern = re.compile(r"\.bdev_aio_(create|delete|rescan)\(") + offenders = [] + for sub in ("simplyblock_core", "simplyblock_web", "simplyblock_cli"): + for path in (root / sub).rglob("*.py"): + if path.name == "rpc_client.py": + continue + for n, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if pattern.search(line): + offenders.append(f"{path.relative_to(root)}:{n}") + self.assertEqual(offenders, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_lblk_journal_only.py b/tests/unit/test_lblk_journal_only.py new file mode 100644 index 0000000000..eb9ec33ef8 --- /dev/null +++ b/tests/unit/test_lblk_journal_only.py @@ -0,0 +1,60 @@ +"""``sn configure --lblk-journal-only``: on a multi-attach cluster whose array +plugin creates each node's data LUN at add-node, the node's one local disk is +its journal and nothing else (never partitioned, no local data device).""" +import unittest + +from simplyblock_core import utils + + +def _disk(name="sdb", **kw): + return dict({"name": name, "serial": f"S-{name}", "size": 50 * 2**30, "type": "disk"}, **kw) + + +class TestJournalOnlyLayout(unittest.TestCase): + + def test_single_disk_becomes_flagged_journal(self): + out = utils.lblk_journal_only_layout({"sdb": _disk()}) + self.assertEqual(list(out), ["sdb"]) + self.assertTrue(out["sdb"]["journal"]) + self.assertEqual(out["sdb"]["serial"], "S-sdb") + + def test_input_is_not_modified(self): + entries = {"sdb": _disk()} + utils.lblk_journal_only_layout(entries) + self.assertNotIn("journal", entries["sdb"]) + + def test_two_disks_refused(self): + with self.assertRaisesRegex(ValueError, "exactly one disk or partition"): + utils.lblk_journal_only_layout({"sdb": _disk(), "sdc": _disk("sdc")}, socket=1) + + def test_no_disk_refused(self): + with self.assertRaisesRegex(ValueError, "exactly one disk or partition"): + utils.lblk_journal_only_layout({}) + + def test_partition_is_a_journal_too(self): + out = utils.lblk_journal_only_layout({"sdb1": _disk("sdb1", type="part")}) + self.assertTrue(out["sdb1"]["journal"]) + + +class TestConfigureArgs(unittest.TestCase): + """The flag is in the CLI reference and the generated CLI.""" + + def test_flag_is_known_to_the_cli_reference(self): + import os + + import yaml + path = os.path.join(os.path.dirname(utils.__file__), "..", "..", "simplyblock_cli", "cli-reference.yaml") + with open(path, encoding="utf-8") as f: + text = f.read() + self.assertIn("--lblk-journal-only", text) + ref = yaml.safe_load(text) + self.assertTrue(ref) + + def test_generated_cli_has_the_flag(self): + from simplyblock_cli import cli + with open(cli.__file__, encoding="utf-8") as f: + self.assertIn("'--lblk-journal-only'", f.read()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_lblk_stable_paths.py b/tests/unit/test_lblk_stable_paths.py index d44ce3604d..0be129dc5c 100644 --- a/tests/unit/test_lblk_stable_paths.py +++ b/tests/unit/test_lblk_stable_paths.py @@ -184,7 +184,7 @@ def test_the_path_is_the_filename_and_the_recorded_by_id_path(self): devices = utils.addAioDevices(rpc, self._snode(), resolved) rpc.bdev_aio_create.assert_called_once_with( - utils.aio_bdev_name_for_serial("S-DATA"), DATA_BY_ID_ALT) + utils.aio_bdev_name_for_serial("S-DATA"), DATA_BY_ID_ALT, block_size=4096) self.assertEqual(devices[0].by_id_path, DATA_BY_ID_ALT) diff --git a/tests/unit/test_lblk_watchdog.py b/tests/unit/test_lblk_watchdog.py index afe7d03017..b7ca035878 100644 --- a/tests/unit/test_lblk_watchdog.py +++ b/tests/unit/test_lblk_watchdog.py @@ -217,6 +217,64 @@ def test_no_aio_devices_no_inventory_call(self): node.client.assert_not_called() +def _ma_dev(serving="", fenced=(), node_id="node-1"): + d = _aio_dev() + d.multi_attach = True + d.node_id = node_id + d.serving_node_id = serving + d.fenced_node_ids = list(fenced) + return d + + +class TestMultiAttachNotLocal(WatchdogBase): + """2026-11-19 FTT1 bed: a multi-attach device whose LUN another node + serves (or whose owner is fenced from it) is not the owner's to sweep: + the fence detached the LUN from the owner's host, and a hot-removal + verdict turned into device_remove and a cluster suspension.""" + + def _absent(self, dev): + node = _node([dev]) + client = MagicMock() + client.get_blockdevices.return_value = ([{"name": "sdz", "serial": "ZZ"}], None) + node.client.return_value = client + return node + + def test_served_elsewhere_never_reported_absent(self): + dev = _ma_dev(serving="node-2") + node = self._absent(dev) + for _ in range(constants.AIO_DEVICE_ABSENT_POLLS + 2): + self.assertEqual(_check_aio_device_presence(node), []) + + def test_fenced_owner_never_reported_absent(self): + dev = _ma_dev(fenced=["node-1"]) + node = self._absent(dev) + for _ in range(constants.AIO_DEVICE_ABSENT_POLLS + 2): + self.assertEqual(_check_aio_device_presence(node), []) + + def test_served_here_still_swept(self): + dev = _ma_dev() + node = self._absent(dev) + for _ in range(constants.AIO_DEVICE_ABSENT_POLLS - 1): + self.assertEqual(_check_aio_device_presence(node), []) + self.assertEqual(_check_aio_device_presence(node), [dev]) + + def test_served_elsewhere_never_stalls(self): + dev = _ma_dev(serving="node-2") + node = _node([dev]) + rpc = _rpc_with_stats({dev.nvme_bdev: _stat(100, 5)}) + for _ in range(constants.AIO_HUNG_IO_STALL_POLLS + 2): + self.assertEqual(_check_aio_hung_io(node, rpc), []) + rpc.get_lvol_stats.assert_not_called() + + def test_repair_not_due_when_served_elsewhere(self): + from simplyblock_core.controllers import device_controller + dev = _ma_dev(serving="node-2") + dev.status = NVMeDevice.STATUS_UNAVAILABLE + self.assertFalse(device_controller.device_repair_due(dev)) + dev.serving_node_id = "" + self.assertTrue(device_controller.device_repair_due(dev)) + + class TestSweepActions(WatchdogBase): def _sweep(self, node, stalled=None, gone=None): diff --git a/tests/unit/test_lun_takeover.py b/tests/unit/test_lun_takeover.py new file mode 100644 index 0000000000..622af8ff35 --- /dev/null +++ b/tests/unit/test_lun_takeover.py @@ -0,0 +1,1243 @@ +"""Multi-attach array LUN mode, phase 3: LUN takeover by a standby node. + + - array plugin interface: registry, entry points, the fake is never + registered by default. + - controllers/lun_takeover: fence first, standby choice, open-only alceml, + export under the same NQN/namespace, record, paths; fence failure, no + plugin, no readable standby, idempotent re-run, concurrent takeover. + - storage_node_monitor: takeovers on node loss (multi-attach only), + unavailable on failure, regular clusters unchanged. + - storage_node_ops: owner restart skips LUNs served elsewhere; discovery + never adopts (formats) another node's LUN; connect uses the serving node. + - device_controller: ONLINE while the owner is down only via a takeover. +""" + +import copy +import threading +import time +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import array, constants, multi_attach, storage_node_ops, utils +from simplyblock_core.array.fake import FakeArrayPlugin +from simplyblock_core.controllers import device_controller, lun_takeover +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.iface import IFace +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import storage_node_monitor + +DEV_ID = "1b4e28ba-2fa1-41d2-883f-0016d3cca427" +WWN = "0x600a098038314b6f2b5d4e6a4f2d6b31" +BY_ID = "/dev/disk/by-id/wwn-0x600a098038314b6f2b5d4e6a4f2d6b31" +SIZE = 8 * 1024 ** 3 +PLUGIN = "fake-test" + + +# ------------------------------------------------------------------ fakes + +class FakeSpdk: + """The RPC surface a takeover uses, with bdev/subsystem state.""" + + def __init__(self, name): + self.name = name + self.bdevs = {} + self.subsystems = {} + self.calls = [] + self.fail = set() + self.down = False + + def get_version(self): + if self.down: + raise ConnectionError("Could not reach remote") + return {"version": "fake"} + + def _rec(self, op, *args, **kwargs): + self.calls.append((op, args, kwargs)) + return op not in self.fail + + def ops(self, op): + return [c for c in self.calls if c[0] == op] + + def bdev_get(self, name): + return copy.deepcopy(self.bdevs.get(name)) + + def bdev_aio_create(self, name, filename, block_size=0): + if not self._rec("bdev_aio_create", name, filename, block_size): + return None + self.bdevs[name] = {"name": name, "block_size": 4096, "num_blocks": SIZE // 4096} + return name + + def bdev_alceml_create(self, name, base, uuid, **kwargs): + if not self._rec("bdev_alceml_create", name, base, uuid, **kwargs): + return None + self.bdevs[name] = {"name": name} + return name + + def bdev_PT_NoExcl_create(self, name, base): + self._rec("bdev_PT_NoExcl_create", name, base) + self.bdevs[name] = {"name": name, "aliases": [f"pt-uuid-{self.name}"]} + return True + + def subsystem_get(self, nqn): + return copy.deepcopy(self.subsystems.get(nqn)) + + def subsystem_create(self, nqn, sn, mn, **kwargs): + self._rec("subsystem_create", nqn, sn, mn, **kwargs) + self.subsystems[nqn] = {"nqn": nqn, "listen_addresses": [], "namespaces": []} + return True + + def listeners_create(self, nqn, trtype, traddr, port): + self._rec("listeners_create", nqn, trtype, traddr, port) + self.subsystems[nqn]["listen_addresses"].append({"traddr": traddr}) + return True + + def nvmf_subsystem_add_ns(self, nqn, pt, uuid=None, nguid=None): + if not self._rec("nvmf_subsystem_add_ns", nqn, pt, uuid): + return None + self.subsystems[nqn]["namespaces"].append({"name": pt, "uuid": uuid, "nguid": nguid}) + return 1 + + def alceml_set_qos_weights(self, weights): + return True + + def bdev_nvme_attach_controller(self, *args, **kwargs): + return self._rec("attach", *args, **kwargs) + + def bdev_nvme_detach_controller(self, *args, **kwargs): + return self._rec("detach", *args, **kwargs) + + def bdev_nvme_set_multipath_policy(self, *args, **kwargs): + return self._rec("policy", *args, **kwargs) + + def subsystem_delete(self, nqn): + self._rec("subsystem_delete", nqn) + self.subsystems.pop(nqn, None) + + def bdev_PT_NoExcl_delete(self, name): + self._rec("bdev_PT_NoExcl_delete", name) + self.bdevs.pop(name, None) + + def bdev_alceml_delete(self, name): + self._rec("bdev_alceml_delete", name) + self.bdevs.pop(name, None) + + def bdev_aio_delete(self, name): + self._rec("bdev_aio_delete", name) + self.bdevs.pop(name, None) + + +class FakeDB: + def __init__(self, cluster, nodes): + self.cluster = cluster + self.nodes = {n.get_id(): n for n in nodes} + self.lock_holder = None + + def get_cluster_by_id(self, _cid): + return self.cluster + + def get_storage_nodes_by_cluster_id(self, _cid): + return list(self.nodes.values()) + + def get_storage_node_by_id(self, nid): + if nid not in self.nodes: + raise KeyError(nid) + return self.nodes[nid] + + def get_storage_device_by_id(self, did): + for n in self.nodes.values(): + for d in n.nvme_devices: + if d.get_id() == did: + return d + raise KeyError(did) + + def atomic_update(self, obj, fn): + fresh = self.nodes[obj.get_id()] + fn(fresh) + return fresh + + def acquire_lun_takeover_lock(self, _did, owner, _ttl): + if self.lock_holder and self.lock_holder != owner: + return False, self.lock_holder + self.lock_holder = owner + return True, None + + def release_lun_takeover_lock(self, _did, owner): + if self.lock_holder == owner: + self.lock_holder = None + + +def _cluster(plugin=PLUGIN, multi=True): + cl = Cluster() + cl.uuid = "cluster-1" + cl.nqn = f"{constants.CLUSTER_NQN}:cluster-1" + cl.distr_ndcs, cl.distr_npcs = 1, 0 + cl.multi_attach = multi + cl.array_plugin = plugin + cl.status = Cluster.STATUS_ACTIVE + return cl + + +def _node(nid, ip, block, status=StorageNode.STATUS_ONLINE): + n = StorageNode() + n.uuid = nid + n.cluster_id = "cluster-1" + n.hostname = f"host-{nid}_5260" + n.rpc_port = 5260 + n.status = status + n.nvmf_port = 4420 + n.cntlid_block = block + n.active_tcp = True + nic = IFace() + nic.ip4_address = ip + n.data_nics = [nic] + n.spdk = FakeSpdk(nid) + n.rpc_client = lambda *a, _s=n.spdk, **k: _s + n.agent = MagicMock() + n.client = lambda *a, _c=n.agent, **k: _c + return n + + +def _lun_entry(signature=True, path=BY_ID, size=SIZE): + return {"name": "sdc", "device_path": "/dev/sdc", "by_id_path": path, + "by_id_paths": [path], "size": size, "has_alceml_signature": signature} + + +def _bed(plugin=PLUGIN, multi=True, dev_status=NVMeDevice.STATUS_ONLINE): + """Owner A (serves its LUN), secondary B, tertiary C, peer D.""" + a = _node("node-a", "10.0.0.1", 0) + b = _node("node-b", "10.0.0.2", 1) + c = _node("node-c", "10.0.0.3", 2) + d = _node("node-d", "10.0.0.4", 3) + a.secondary_node_id, a.tertiary_node_id = "node-b", "node-c" + dev = NVMeDevice() + dev.uuid = DEV_ID + dev.node_id = "node-a" + dev.cluster_id = "cluster-1" + dev.status = dev_status + dev.size = SIZE + dev.bdev_type = "aio" + dev.serial_number = WWN + dev.multi_attach = multi + dev.lun_wwn = WWN + dev.alceml_bdev = dev.alceml_name = f"alceml_{DEV_ID}" + dev.pt_bdev = f"alceml_{DEV_ID}_PT" + dev.nvmf_nqn = multi_attach.device_nqn(f"{constants.CLUSTER_NQN}:cluster-1", DEV_ID) + dev.nvmf_ip = "10.0.0.1" + dev.nvmf_port = 4420 + dev.node_paths = {"node-a": {"path": BY_ID, "bdev": "aio_x"}} + a.nvme_devices = [dev] + for n in (b, c, d): + n.agent.lun_info.return_value = ([_lun_entry()], None) + a.status = StorageNode.STATUS_OFFLINE + return FakeDB(_cluster(plugin, multi), [a, b, c, d]), dev + + +class _TakeoverCase(unittest.TestCase): + + def setUp(self): + FakeArrayPlugin.reset() + array.register_plugin(PLUGIN, FakeArrayPlugin) + self.addCleanup(array.unregister_plugin, PLUGIN) + array.forget_plugin() + self.addCleanup(array.forget_plugin) + self.events = [] + for target, attr, kw in ( + (lun_takeover, "DBController", {}), + (lun_takeover.ec, "log_event_cluster", + {"side_effect": lambda **k: self.events.append(k)}), + (device_controller, "device_set_state", {"return_value": True})): + p = patch.object(target, attr, **kw) + setattr(self, f"m_{attr}", p.start()) + self.addCleanup(p.stop) + qos = patch("simplyblock_core.controllers.qos_controller.get_qos_weights_list", return_value=[]) + qos.start() + self.addCleanup(qos.stop) + + def run_takeover(self, db, failed="node-a", **kw): + self.m_DBController.return_value = db + return lun_takeover.take_over_lun_outcome(DEV_ID, failed, "test", **kw) + + def dev(self, db): + return db.get_storage_device_by_id(DEV_ID) + + +# ------------------------------------------------------------ the takeover + +class TestTakeover(_TakeoverCase): + + def test_happy_path(self): + db, _ = _bed() + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + # 1. fence of the failed node, by hostname, first. + self.assertEqual(FakeArrayPlugin.calls, [("fence", WWN, "host-node-a")]) + b = db.nodes["node-b"] + # 4. open-only alceml on the secondary, over the by-id path. + aio = b.spdk.ops("bdev_aio_create") + self.assertEqual(aio[0][1][1], BY_ID) + alceml = b.spdk.ops("bdev_alceml_create") + self.assertEqual(len(alceml), 1) + self.assertEqual(alceml[0][2]["pba_init_mode"], 1) + self.assertEqual(alceml[0][2]["stuck_io_timeout_ms"], constants.MULTI_ATTACH_ALCEML_STUCK_IO_MS) + nqn = multi_attach.device_nqn(db.cluster.nqn, DEV_ID) + sub = b.spdk.ops("subsystem_create")[0] + self.assertEqual(sub[1][0], nqn) + self.assertEqual((sub[2]["min_cntlid"], sub[2]["max_cntlid"]), multi_attach.cntlid_range(1)) + ns = b.spdk.ops("nvmf_subsystem_add_ns")[0] + self.assertEqual(ns[1][2], multi_attach.namespace_uuid(self.dev(db))) + self.assertEqual(b.spdk.ops("listeners_create")[0][1][2], "10.0.0.2") + # 6. record. + dev = self.dev(db) + self.assertEqual(dev.serving_node(), "node-b") + self.assertEqual(dev.nvmf_ip, "10.0.0.2") + self.assertEqual(dev.node_paths["node-b"]["path"], BY_ID) + self.assertIn("node-a", dev.fenced_node_ids) + self.assertEqual(dev.status, NVMeDevice.STATUS_ONLINE) + self.m_device_set_state.assert_not_called() + # 5 + 7. new path on every ONLINE node (standby's loopback too), old one off. + for nid in ("node-b", "node-c", "node-d"): + spdk = db.nodes[nid].spdk + att = spdk.ops("attach") + self.assertEqual(len(att), 1, nid) + self.assertEqual(att[0][1][:5], (f"remote_alceml_{DEV_ID}", nqn, "10.0.0.2", 4420, "TCP")) + self.assertEqual(att[0][2]["multipath"], "multipath") + self.assertEqual(att[0][2]["ctrlr_loss_timeout_sec"], constants.MULTI_ATTACH_HOLD_SEC) + self.assertEqual(spdk.ops("policy")[0][1], (f"remote_alceml_{DEV_ID}n1", "active_passive")) + det = spdk.ops("detach") + self.assertEqual(det[0][2]["traddr"], "10.0.0.1") + self.assertEqual(db.nodes["node-a"].spdk.calls, []) + self.assertTrue(any("taken over" in e["message"] for e in self.events)) + + def test_never_formats(self): + db, _ = _bed() + self.run_takeover(db) + for n in db.nodes.values(): + for call in n.spdk.ops("bdev_alceml_create"): + self.assertEqual(call[2]["pba_init_mode"], lun_takeover.ALCEML_OPEN_ONLY) + + def test_fence_failure_means_no_takeover(self): + for mode in ("fail", "fail_falsy"): + with self.subTest(mode=mode): + FakeArrayPlugin.reset() + getattr(FakeArrayPlugin, mode).add("fence") + db, _ = _bed() + self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) + self.assertEqual(self.dev(db).serving_node(), "node-a") + # The standby may have been probed (read-only, during the + # fence); nothing was built on any node. + for n in db.nodes.values(): + self.assertEqual(n.spdk.calls, []) + + def test_no_plugin_refuses(self): + db, _ = _bed(plugin="") + self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) + self.assertEqual(self.dev(db).serving_node(), "node-a") + self.assertTrue(all(n.spdk.calls == [] for n in db.nodes.values())) + self.assertTrue(any("no array plugin" in e["message"] for e in self.events)) + + def test_unknown_plugin_refuses(self): + db, _ = _bed(plugin="no-such-plugin") + self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) + self.assertEqual(FakeArrayPlugin.calls, []) + + def test_no_standby_can_read_holds(self): + db, _ = _bed() + for nid in ("node-b", "node-c"): + db.nodes[nid].agent.lun_info.return_value = ([_lun_entry(signature=None)], None) + self.assertEqual(self.run_takeover(db), lun_takeover.NO_STANDBY) + self.assertEqual(self.dev(db).serving_node(), "node-a") + self.assertTrue(all(n.spdk.calls == [] for n in db.nodes.values())) + # The node that is not owner/secondary/tertiary is never a candidate. + db.nodes["node-d"].agent.lun_info.assert_not_called() + self.assertTrue(any("no standby can read" in e["message"] for e in self.events)) + + def test_secondary_offline_uses_tertiary(self): + db, _ = _bed() + db.nodes["node-b"].status = StorageNode.STATUS_OFFLINE + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).serving_node(), "node-c") + self.assertEqual(self.dev(db).nvmf_ip, "10.0.0.3") + self.assertEqual(db.nodes["node-b"].spdk.calls, []) + + def test_secondary_unreadable_uses_tertiary(self): + db, _ = _bed() + db.nodes["node-b"].agent.lun_info.return_value = ([], None) + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).serving_node(), "node-c") + + def test_rerun_is_idempotent(self): + db, _ = _bed() + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + for n in db.nodes.values(): + n.spdk.calls.clear() + FakeArrayPlugin.calls.clear() + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(FakeArrayPlugin.calls, []) + b = db.nodes["node-b"] + self.assertEqual(b.spdk.ops("bdev_alceml_create"), []) + self.assertEqual(b.spdk.ops("subsystem_create"), []) + self.assertEqual(self.dev(db).serving_node(), "node-b") + self.assertTrue(b.spdk.ops("attach")) + + def test_path_switch_marks_the_device_online_on_every_node(self): + # A node told "unavailable" during the gap must not keep it (1+0: + # every read there would fail). + from simplyblock_core import distr_controller + db, _ = _bed() + with patch.object(distr_controller, "send_dev_status_event") as sent: + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + targets = sorted(c.args[2].get_id() for c in sent.call_args_list + if c.args[1] == NVMeDevice.STATUS_ONLINE) + self.assertEqual(targets, sorted(n for n, x in db.nodes.items() + if n != "node-a" and x.status == StorageNode.STATUS_ONLINE)) + + def test_rerun_after_partial_export_reuses_it(self): + db, _ = _bed() + standbys = [db.nodes[n] for n in ("node-b", "node-c", "node-d")] + for n in standbys: + n.spdk.fail.add("nvmf_subsystem_add_ns") + self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) + # Cleanup closed what each attempt opened; nothing recorded. + self.assertEqual(self.dev(db).serving_node(), "node-a") + b = standbys[0] + self.assertNotIn(f"alceml_{DEV_ID}", b.spdk.bdevs) + self.assertTrue(b.spdk.ops("bdev_alceml_delete")) + for n in standbys: + n.spdk.fail.clear() + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).serving_node(), "node-b") + + def test_failed_export_moves_to_the_next_standby(self): + db, _ = _bed() + db.nodes["node-b"].spdk.fail.add("nvmf_subsystem_add_ns") + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).serving_node(), "node-c") + # node-b was retried once (it answers), then left; its stack closed. + self.assertEqual(len(db.nodes["node-b"].spdk.ops("bdev_alceml_create")), 2) + self.assertNotIn(f"alceml_{DEV_ID}", db.nodes["node-b"].spdk.bdevs) + + def test_standby_is_chosen_while_the_fence_runs(self): + import threading + db, _ = _bed() + probed = threading.Event() + db.nodes["node-b"].agent.lun_info.side_effect = ( + lambda *_a, **_k: (probed.set(), ([_lun_entry()], None))[1]) + seen = [] + + class SlowFence(FakeArrayPlugin): + def fence(self, lun_id, host_id): + seen.append(probed.wait(5)) + return super().fence(lun_id, host_id) + + array.register_plugin(PLUGIN, SlowFence) + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(seen, [True]) + + def test_nothing_is_built_before_the_fence_is_acknowledged(self): + db, _ = _bed() + order = [] + b = db.nodes["node-b"] + create = b.spdk.bdev_aio_create + b.spdk.bdev_aio_create = lambda *a, **k: (order.append("build"), create(*a, **k))[1] + + class Fence(FakeArrayPlugin): + def fence(self, lun_id, host_id): + order.append("fence") + return super().fence(lun_id, host_id) + + array.register_plugin(PLUGIN, Fence) + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(order, ["fence", "build"]) + + def test_array_connection_is_reused(self): + db, _ = _bed() + self.run_takeover(db) + db.nodes["node-b"].status = StorageNode.STATUS_OFFLINE + self.assertEqual(self.run_takeover(db, failed="node-b"), lun_takeover.TAKEN_OVER) + self.assertEqual(len(FakeArrayPlugin.connected), 1) + + def test_fence_failure_retried_on_a_new_connection(self): + db, _ = _bed() + stale, fresh = MagicMock(), MagicMock() + stale.fence.side_effect = RuntimeError("ExpiredToken") + fresh.fence.return_value = True + with patch.object(array, "cached_plugin_for_cluster", side_effect=[stale, fresh]), patch.object(array, "forget_plugin") as forget: + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + forget.assert_called_once_with("cluster-1") + fresh.fence.assert_called_once_with(WWN, "host-node-a") + + def test_concurrent_takeover_is_refused(self): + db, _ = _bed() + db.lock_holder = "another-runner" + self.assertEqual(self.run_takeover(db), lun_takeover.IN_PROGRESS) + self.assertEqual(FakeArrayPlugin.calls, []) + + def test_failed_standby_is_taken_over_again(self): + db, _ = _bed() + self.run_takeover(db) + db.nodes["node-b"].status = StorageNode.STATUS_OFFLINE + FakeArrayPlugin.calls.clear() + self.assertEqual(self.run_takeover(db, failed="node-b"), lun_takeover.TAKEN_OVER) + self.assertEqual(FakeArrayPlugin.calls, [("fence", WWN, "host-node-b")]) + dev = self.dev(db) + self.assertEqual(dev.serving_node(), "node-c") + self.assertEqual(sorted(dev.fenced_node_ids), ["node-a", "node-b"]) + + def test_unavailable_device_comes_back_online(self): + db, _ = _bed(dev_status=NVMeDevice.STATUS_UNAVAILABLE) + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.m_device_set_state.assert_called_once_with( + DEV_ID, NVMeDevice.STATUS_ONLINE, cause=device_controller.CAUSE_LUN_TAKEOVER, + connect_peers=False) + + def test_unfence_node(self): + db, _ = _bed() + self.run_takeover(db) + self.assertTrue(lun_takeover.unfence_node(DEV_ID, "node-a")) + self.assertEqual(FakeArrayPlugin.calls[-1], ("unfence", WWN, "host-node-a")) + self.assertEqual(self.dev(db).fenced_node_ids, []) + # The serving node is never "unfenced" (it was never fenced). + self.assertFalse(lun_takeover.unfence_node(DEV_ID, "node-b")) + + def test_regular_device_is_refused(self): + db, _ = _bed(multi=False) + self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) + self.assertEqual(FakeArrayPlugin.calls, []) + + def test_size_mismatch_is_refused(self): + db, dev = _bed() + dev.size = SIZE * 2 + for nid in ("node-b", "node-c"): + db.nodes[nid].agent.lun_info.return_value = ([_lun_entry(size=SIZE * 2)], None) + self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) + self.assertEqual(self.dev(db).serving_node(), "node-a") + + +# ----------------------------------------------------------- helpers/array + +class TestHelpers(unittest.TestCase): + + def test_pick_lun_path(self): + dev = NVMeDevice() + dev.node_id = "a" + dev.size = SIZE + dev.node_paths = {"a": {"path": BY_ID}} + self.assertEqual(multi_attach.pick_lun_path([_lun_entry()], dev), (BY_ID, "")) + self.assertIsNone(multi_attach.pick_lun_path([], dev)[0]) + self.assertIsNone(multi_attach.pick_lun_path([_lun_entry(signature=False)], dev)[0]) + self.assertIsNone(multi_attach.pick_lun_path([_lun_entry(signature=None)], dev)[0]) + other = _lun_entry(path="/dev/disk/by-id/scsi-x") + self.assertIsNone(multi_attach.pick_lun_path([other, dict(other, name="sdd")], dev)[0]) + self.assertEqual(multi_attach.pick_lun_path([other, _lun_entry()], dev)[0], BY_ID) + + def test_array_host_and_candidates(self): + n = StorageNode() + n.uuid = "a" + n.hostname, n.rpc_port = "worker-1_5260", 5260 + self.assertEqual(multi_attach.array_host(n), "worker-1") + n.secondary_node_id, n.tertiary_node_id = "b", "c" + self.assertEqual(multi_attach.standby_candidates(n, "a"), ["b", "c"]) + self.assertEqual(multi_attach.standby_candidates(n, "b"), ["c", "a"]) + + def test_fake_plugin_is_not_registered_by_default(self): + with self.assertRaises(array.ArrayError): + array.plugin_class("fake") + with self.assertRaises(array.ArrayNotConfigured): + array.plugin_for_cluster(_cluster(plugin="")) + + def test_entry_point_loading(self): + ep = MagicMock(value="pkg:Cls") + ep.load.return_value = FakeArrayPlugin + with patch.object(array.metadata, "entry_points", return_value=[ep]) as eps: + plugin = array.load_plugin("vendor", "https://array", {"pool": "p"}) + eps.assert_called_once_with(group=array.ENTRY_POINT_GROUP, name="vendor") + self.assertIsInstance(plugin, FakeArrayPlugin) + self.assertEqual((plugin.endpoint, plugin.options), ("https://array", {"pool": "p"})) + ep.load.return_value = object + with patch.object(array.metadata, "entry_points", return_value=[ep]): + with self.assertRaises(array.ArrayError): + array.load_plugin("vendor") + + def test_find_lun_devices(self): + from simplyblock_web import node_utils + inv = [{"type": "disk", "name": "sdb", "device_path": "/dev/sdb", "wwn": "0X600A", + "serial": "s1", "size": 1, "by_id_path": "/x"}, + {"type": "disk", "name": "sdc", "device_path": "/dev/sdc", "wwn": "", "serial": "zz", + "size": 1, "by_id_path": "/y"}] + with patch.object(node_utils, "get_block_devices_info", return_value=inv), \ + patch.object(node_utils, "simplyblock_data_signature", return_value=True) as sig: + out = node_utils.find_lun_devices("600a") + self.assertEqual([e["name"] for e in out], ["sdb"]) + self.assertTrue(out[0]["has_alceml_signature"]) + sig.assert_called_once_with("sdb") + + def test_find_lun_devices_skips_a_dead_controller(self): + # A volume detached while the host booted: the kernel keeps a 0-byte + # namespace of a dead controller. + from simplyblock_web import node_utils + inv = [{"type": "disk", "name": "nvme3n1", "device_path": "/dev/nvme3n1", "wwn": "", + "serial": "vol08dcbe488933595ca", "size": 0, "by_id_path": "/x"}] + states = {"/sys/block/nvme3n1/device/state": "dead"} + with patch.object(node_utils, "get_block_devices_info", return_value=inv), patch.object(node_utils, "_read_sysfs", side_effect=lambda p: states.get(p, "")), patch.object(node_utils, "simplyblock_data_signature", return_value=None): + self.assertEqual(node_utils.find_lun_devices("vol08dcbe488933595ca"), []) + states["/sys/block/nvme3n1/device/state"] = "live" + self.assertEqual(len(node_utils.find_lun_devices("vol08dcbe488933595ca")), 1) + + +# ------------------------------------------------------------------ monitor + +class TestMonitorHook(unittest.TestCase): + + def _run(self, db): + started = [] + with patch.object(storage_node_monitor, "db", db), \ + patch.object(storage_node_monitor, "_start_lun_takeover", + side_effect=lambda did, nid, why: started.append((did, nid))), \ + patch.object(storage_node_monitor.device_controller, + "device_set_unavailable") as unavailable: + storage_node_monitor._set_node_devices_unavailable(db.nodes["node-a"], "test") + return started, [c.args[0] for c in unavailable.call_args_list] + + def test_multi_attach_starts_takeover_instead_of_unavailable(self): + db, dev = _bed() + plain = NVMeDevice() + plain.uuid, plain.node_id, plain.status = "plain", "node-a", NVMeDevice.STATUS_ONLINE + db.nodes["node-a"].nvme_devices.append(plain) + started, unavailable = self._run(db) + self.assertEqual(started, [(DEV_ID, "node-a")]) + self.assertEqual(unavailable, ["plain"]) + + def test_lun_served_elsewhere_is_untouched_and_taken_over_luns_follow(self): + db, dev = _bed() + dev.serving_node_id = "node-b" + foreign = copy.deepcopy(dev) + foreign.uuid, foreign.node_id, foreign.serving_node_id = "foreign", "node-c", "node-a" + db.nodes["node-c"].nvme_devices = [foreign] + started, unavailable = self._run(db) + self.assertEqual(started, [("foreign", "node-a")]) + self.assertEqual(unavailable, []) + + def test_regular_cluster_unchanged(self): + db, dev = _bed(multi=False) + started, unavailable = self._run(db) + self.assertEqual(started, []) + self.assertEqual(unavailable, [DEV_ID]) + + def test_worker_marks_unavailable_on_failure_only(self): + db, dev = _bed() + for outcome, expected in ((lun_takeover.FAILED, [DEV_ID]), + (lun_takeover.TAKEN_OVER, []), + (lun_takeover.IN_PROGRESS, [])): + with self.subTest(outcome=outcome), \ + patch.object(storage_node_monitor, "db", db), \ + patch.object(lun_takeover, "take_over_lun_outcome", return_value=outcome), \ + patch.object(storage_node_monitor.device_controller, + "device_set_unavailable") as unavailable: + storage_node_monitor._lun_takeover_worker(DEV_ID, "node-a", "t", 0.0) + self.assertEqual([c.args[0] for c in unavailable.call_args_list], expected) + + def test_worker_holds_until_the_hold_expires(self): + db, dev = _bed() + with patch.object(storage_node_monitor, "db", db), \ + patch.object(lun_takeover, "take_over_lun_outcome", return_value=lun_takeover.NO_STANDBY), \ + patch.object(storage_node_monitor.time, "monotonic", return_value=100.0), \ + patch.object(storage_node_monitor.time, "sleep") as sleep, \ + patch.object(storage_node_monitor.device_controller, + "device_set_unavailable") as unavailable: + storage_node_monitor._lun_takeover_worker(DEV_ID, "node-a", "t", 90.0) + sleep.assert_called_once_with(90.0 + constants.MULTI_ATTACH_HOLD_SEC - 100.0) + unavailable.assert_called_once_with(DEV_ID) + + +# ----------------------------------------------------- owner restart/discovery + +class TestOwnerReturn(unittest.TestCase): + + def test_restart_skips_luns_served_elsewhere(self): + db, dev = _bed() + dev.serving_node_id = "node-b" + own = copy.deepcopy(dev) + own.uuid, own.serving_node_id = "own-served", "" + a = db.nodes["node-a"] + a.nvme_devices = [dev, own] + a.jm_device = None + built = [] + with patch.object(storage_node_ops, "DBController"), \ + patch.object(storage_node_ops, "_create_storage_device_stack", + side_effect=lambda rpc, nvme, sn, ar: built.append(nvme.get_id()) or nvme), \ + patch.object(StorageNode, "write_to_db"): + self.assertTrue(storage_node_ops._prepare_cluster_devices_on_restart(a)) + self.assertEqual(built, ["own-served"]) + self.assertEqual([d.get_id() for d in a.nvme_devices], [DEV_ID, "own-served"]) + + def test_restart_skips_luns_it_is_fenced_from(self): + db, dev = _bed(dev_status=NVMeDevice.STATUS_UNAVAILABLE) + dev.fenced_node_ids = ["node-a"] + a = db.nodes["node-a"] + a.jm_device = None + with patch.object(storage_node_ops, "DBController"), patch.object(storage_node_ops, "_create_storage_device_stack") as build, patch.object(StorageNode, "write_to_db"): + self.assertTrue(storage_node_ops._prepare_cluster_devices_on_restart(a)) + build.assert_not_called() + + def _reconcile(self, db, node, found): + with patch.object(storage_node_ops, "DBController", return_value=db): + return storage_node_ops._reconcile_discovered_devices(node, found) + + def _found(self, serial, wwn=""): + d = NVMeDevice() + d.uuid, d.serial_number, d.lun_wwn = f"new-{serial}", serial, wwn + d.device_name = "sdx" + return d + + def test_discovery_never_adopts_another_nodes_lun(self): + db, dev = _bed() + b = db.nodes["node-b"] + b.nvme_devices = [] + new = self._reconcile(db, b, [self._found(WWN.upper(), WWN), self._found("own-lun")]) + self.assertEqual([d.serial_number for d in new], ["own-lun"]) + self.assertNotIn(WWN.upper(), [d.serial_number for d in b.nvme_devices]) + + def test_missing_lun_served_elsewhere_is_kept(self): + db, dev = _bed() + dev.serving_node_id = "node-b" + a = db.nodes["node-a"] + self.assertEqual(self._reconcile(db, a, []), []) + self.assertEqual(dev.status, NVMeDevice.STATUS_ONLINE) + + def test_regular_cluster_discovery_unchanged(self): + db, dev = _bed(multi=False) + b = db.nodes["node-b"] + b.nvme_devices = [] + new = self._reconcile(db, b, [self._found(WWN, WWN)]) + self.assertEqual(len(new), 1) + a = db.nodes["node-a"] + self.assertEqual(self._reconcile(db, a, []), []) + self.assertEqual(dev.status, NVMeDevice.STATUS_REMOVED) + + def test_add_node_drops_foreign_luns(self): + db, dev = _bed() + with patch.object(storage_node_ops, "DBController", return_value=db): + kept = storage_node_ops.drop_foreign_luns(db.nodes["node-b"], [self._found(WWN, WWN), + self._found("mine")]) + self.assertEqual([d.serial_number for d in kept], ["mine"]) + + +# --------------------------------------------------------- connect / state + +class TestConnectAndState(unittest.TestCase): + + def test_connect_uses_the_serving_node(self): + db, dev = _bed() + dev.serving_node_id = "node-b" + dev.nvmf_ip = "10.0.0.2" + db.nodes["node-a"].active_tcp = False # the dead owner: must not matter + db.nodes["node-b"].active_rdma = True + rpc = MagicMock() + rpc.bdev_nvme_controller_list.return_value = [] + rpc.bdev_get.return_value = {"name": "b"} + attach = MagicMock() + attach.bdev_nvme_attach_controller.return_value = [f"remote_alceml_{DEV_ID}n1"] + with patch.object(storage_node_ops, "DBController", return_value=db): + storage_node_ops._connect_device_attach( + f"remote_alceml_{DEV_ID}", dev, db.nodes["node-d"], rpc, attach, ["10.0.0.2"], True) + call = attach.bdev_nvme_attach_controller.call_args + self.assertEqual(call.args[2:5], ("10.0.0.2", 4420, "RDMA")) + rpc.bdev_nvme_set_multipath_policy.assert_called_once_with( + f"remote_alceml_{DEV_ID}n1", "active_passive") + + def test_connect_adds_the_missing_path_to_a_held_controller(self): + db, dev = _bed() + dev.serving_node_id = "node-b" + dev.nvmf_ip = "10.0.0.2" + held = [{"ctrlrs": [{"state": "resetting", "trid": {"traddr": "10.0.0.1"}}]}] + rpc = MagicMock() + rpc.bdev_nvme_controller_list.return_value = held + rpc.bdev_get.return_value = {"name": "b"} + attach = MagicMock() + attach.bdev_nvme_attach_controller.return_value = True + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops.time, "sleep") as sleep: + out = storage_node_ops._connect_device_attach( + f"remote_alceml_{DEV_ID}", dev, db.nodes["node-d"], rpc, attach, ["10.0.0.2"], True) + sleep.assert_not_called() + self.assertEqual(out, f"remote_alceml_{DEV_ID}n1") + self.assertEqual(attach.bdev_nvme_attach_controller.call_args.args[2], "10.0.0.2") + + def _set_online(self, db, cause): + with patch.object(device_controller, "DBController", return_value=db), \ + patch.object(device_controller, "_atomic_device_set"), \ + patch.object(device_controller.device_events, "device_status_change"), \ + patch.object(device_controller.distr_controller, "send_dev_status_event"): + return device_controller.device_set_state( + DEV_ID, NVMeDevice.STATUS_ONLINE, cause=cause, connect_peers=False) + + def test_online_while_owner_down_only_via_takeover(self): + db, dev = _bed(dev_status=NVMeDevice.STATUS_UNAVAILABLE) + dev.serving_node_id = "node-b" + self.assertFalse(self._set_online(db, device_controller.CAUSE_OTHER)) + self.assertTrue(self._set_online(db, device_controller.CAUSE_LUN_TAKEOVER)) + + def test_never_online_while_the_serving_node_is_down_or_fenced(self): + db, dev = _bed(dev_status=NVMeDevice.STATUS_UNAVAILABLE) + db.nodes["node-a"].status = StorageNode.STATUS_ONLINE + dev.serving_node_id = "node-b" + db.nodes["node-b"].status = StorageNode.STATUS_OFFLINE + self.assertFalse(self._set_online(db, device_controller.CAUSE_NODE_RECOVERY)) + dev.serving_node_id = "" + dev.fenced_node_ids = ["node-a"] + self.assertFalse(self._set_online(db, device_controller.CAUSE_OTHER)) + dev.fenced_node_ids = [] + self.assertTrue(self._set_online(db, device_controller.CAUSE_OTHER)) + + def test_takeover_cause_needs_an_online_serving_node(self): + db, dev = _bed(dev_status=NVMeDevice.STATUS_UNAVAILABLE) + self.assertFalse(self._set_online(db, device_controller.CAUSE_LUN_TAKEOVER)) + dev.serving_node_id = "node-b" + db.nodes["node-b"].status = StorageNode.STATUS_OFFLINE + self.assertFalse(self._set_online(db, device_controller.CAUSE_LUN_TAKEOVER)) + + +if __name__ == "__main__": + unittest.main() + + +# --------------------------------------------- multi-attach bed findings, F1 + +class TestUnreachableTrigger(unittest.TestCase): + """The regular monitor starts the takeover as soon as it sees a node + UNREACHABLE (backstop of services/lun_watchdog), not when it is declared + OFFLINE: on the bed the OFFLINE verdict took ~6 min, far past the peers' + hold. No confirmation wait: the fence makes an early takeover safe.""" + + def setUp(self): + from simplyblock_core.services import storage_node_monitor as mon + self.mon = mon + self.db, self.dev = _bed() + self.db.get_storage_node_by_id("node-a").status = StorageNode.STATUS_UNREACHABLE + for target, attr, kw in ((mon, "db", {"new": self.db}), + (mon, "_start_lun_takeover", {})): + p = patch.object(target, attr, **kw) + started = p.start() + if attr == "_start_lun_takeover": + self.start = started + self.addCleanup(p.stop) + + def test_at_once(self): + self.mon._take_over_luns_of_unreachable(self.db.get_storage_node_by_id("node-a")) + self.start.assert_called_once_with(DEV_ID, "node-a", "node unreachable") + + def test_regular_cluster_never(self): + self.db.cluster.multi_attach = False + self.mon._take_over_luns_of_unreachable(self.db.get_storage_node_by_id("node-a")) + self.start.assert_not_called() + + def test_no_confirmation_wait_left(self): + self.assertFalse(hasattr(constants, "MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC")) + self.assertFalse(hasattr(self.mon, "_unreachable_since")) + + def test_hold_covers_detection_and_takeover(self): + # ~20 s for the regular monitor to see the node unreachable, then the + # takeover deadline. + self.assertGreater(constants.MULTI_ATTACH_HOLD_SEC, + 20 + constants.MULTI_ATTACH_TAKEOVER_DEADLINE_SEC) + + +class TestRetireStaleLocalStack(unittest.TestCase): + """A node back ONLINE without a restart drops its stale stack of a LUN + another node took over: own loopback path first, then subsystem, PT, + alceml. Nothing is formatted.""" + + def _setup(self, served_by="node-b", with_stack=True): + db, dev = _bed() + a = db.get_storage_node_by_id("node-a") + a.status = StorageNode.STATUS_ONLINE + dev.serving_node_id = served_by + if with_stack: + a.spdk.bdevs[dev.alceml_bdev] = {"name": dev.alceml_bdev} + a.spdk.bdevs[f"{dev.alceml_bdev}_PT"] = {"name": f"{dev.alceml_bdev}_PT"} + with patch("simplyblock_core.db_controller.DBController", return_value=db): + retired = lun_takeover.retire_stale_local_stack(a) + return a, dev, retired + + def test_retires_in_order(self): + a, dev, retired = self._setup() + self.assertEqual(retired, [DEV_ID]) + detach = a.spdk.ops("detach") + self.assertEqual(detach[0][1][0], f"remote_{dev.alceml_bdev}") + self.assertEqual(detach[0][2]["traddr"], "10.0.0.1") + order = [c[0] for c in a.spdk.calls if c[0] in ("subsystem_delete", "bdev_PT_NoExcl_delete", + "bdev_alceml_delete")] + self.assertEqual(order, ["subsystem_delete", "bdev_PT_NoExcl_delete", "bdev_alceml_delete"]) + self.assertFalse(a.spdk.ops("bdev_alceml_create")) + + def test_nothing_when_still_served_here(self): + a, _dev, retired = self._setup(served_by="") + self.assertEqual(retired, []) + self.assertFalse(a.spdk.ops("subsystem_delete")) + + def test_nothing_without_a_stack(self): + a, _dev, retired = self._setup(with_stack=False) + self.assertEqual(retired, []) + self.assertFalse(a.spdk.ops("detach")) + + +class TestReintegrateFencedNode(_TakeoverCase): + """A node a takeover fenced becomes a standby again (unfenced at the array) + once it is ONLINE, holds no stack of the LUN, and its host shows no disk of + the LUN; otherwise it stays fenced.""" + + def _taken_over(self): + db, _ = _bed() + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + a = db.get_storage_node_by_id("node-a") + a.status = StorageNode.STATUS_ONLINE + a.agent.lun_info.return_value = ([], None) + return db, a + + def test_unfenced_when_host_shows_no_lun(self): + db, a = self._taken_over() + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), [DEV_ID]) + self.assertEqual(FakeArrayPlugin.calls[-1], ("unfence", WWN, "host-node-a")) + self.assertEqual(self.dev(db).fenced_node_ids, []) + self.assertEqual(self.dev(db).serving_node(), "node-b") + + def test_stays_fenced_while_host_shows_lun(self): + db, a = self._taken_over() + a.agent.lun_info.return_value = ([_lun_entry()], None) + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), []) + self.assertEqual(self.dev(db).fenced_node_ids, ["node-a"]) + + def test_stays_fenced_when_inventory_fails(self): + db, a = self._taken_over() + a.agent.lun_info.return_value = (None, "boom") + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), []) + self.assertEqual(self.dev(db).fenced_node_ids, ["node-a"]) + + def test_stays_fenced_while_serving_node_is_down(self): + db, a = self._taken_over() + db.get_storage_node_by_id("node-b").status = StorageNode.STATUS_UNREACHABLE + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), []) + self.assertEqual(self.dev(db).fenced_node_ids, ["node-a"]) + + def test_not_while_node_is_not_online(self): + db, a = self._taken_over() + a.status = StorageNode.STATUS_UNREACHABLE + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), []) + + def test_stale_stack_is_retired_first(self): + db, a = self._taken_over() + dev = self.dev(db) + a.spdk.bdevs[dev.alceml_bdev] = {"name": dev.alceml_bdev} + a.spdk.bdevs[f"{dev.alceml_bdev}_PT"] = {"name": f"{dev.alceml_bdev}_PT"} + with patch("simplyblock_core.db_controller.DBController", return_value=db): + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), [DEV_ID]) + self.assertTrue(a.spdk.ops("bdev_alceml_delete")) + self.assertEqual(self.dev(db).fenced_node_ids, []) + + + def test_stays_fenced_while_its_array_fence_runs(self): + # 2026-11-20 resize bed, R3: unfenced while the background detach + # ran; the detach then left the node off the LUN for good. + db, a = self._taken_over() + key = (DEV_ID, "node-a") + with lun_takeover._fences_guard: + lun_takeover._fences_in_flight.add(key) + try: + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), []) + finally: + with lun_takeover._fences_guard: + lun_takeover._fences_in_flight.discard(key) + self.assertEqual(self.dev(db).fenced_node_ids, ["node-a"]) + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), [DEV_ID]) + + def test_stays_fenced_when_spdk_does_not_answer(self): + # A node just lost can still read ONLINE; its dead SPDK proves + # nothing about a stale stack. + db, a = self._taken_over() + a.spdk.bdev_get = MagicMock(side_effect=ConnectionError("refused")) + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), []) + self.assertEqual(self.dev(db).fenced_node_ids, ["node-a"]) + a.agent.lun_info.assert_not_called() + + +class TestFenceAttachmentRecord(_TakeoverCase): + """An array fence drops the node from array_attached_node_ids, so the + attachment reconcile presents the LUN again once it is unfenced.""" + + def test_array_fence_forgets_the_attachment(self): + db, dev = _bed() + dev.array_attached_node_ids = ["node-a", "node-b", "node-c"] + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).array_attached_node_ids, ["node-b", "node-c"]) + + def test_background_fence_is_in_flight_until_done(self): + db, dev = _bed() + dev.lun_reservations = True + dev.array_attached_node_ids = ["node-a", "node-b"] + gate = threading.Event() + fence = FakeArrayPlugin.fence + + def slow(plugin, *a): + gate.wait(5) + return fence(plugin, *a) + b = db.nodes["node-b"] + b.spdk.alceml_get_role = lambda name: {"role": "active", "state": "active"} + with patch.object(FakeArrayPlugin, "fence", slow): + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertTrue(lun_takeover.fence_in_flight(DEV_ID, "node-a")) + gate.set() + for _ in range(250): + if not lun_takeover.fence_in_flight(DEV_ID, "node-a"): + break + time.sleep(0.02) + self.assertFalse(lun_takeover.fence_in_flight(DEV_ID, "node-a")) + self.assertEqual(self.dev(db).array_attached_node_ids, ["node-b"]) + + +class TestUnfenceOrder(_TakeoverCase): + """The fence record is cleared before the array call: the attachment + reconcile detaches any host the record still lists as fenced.""" + + def test_record_cleared_before_array_attach(self): + db, _ = _bed() + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + seen = [] + orig = FakeArrayPlugin.unfence + + def _unfence(plugin, lun, host): + seen.append(list(self.dev(db).fenced_node_ids)) + return orig(plugin, lun, host) + with patch.object(FakeArrayPlugin, "unfence", _unfence): + self.assertTrue(lun_takeover.unfence_node(DEV_ID, "node-a")) + self.assertEqual(seen, [[]]) + + def test_array_failure_keeps_record_cleared(self): + db, _ = _bed() + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + with patch.object(FakeArrayPlugin, "unfence", side_effect=RuntimeError("api")): + self.assertTrue(lun_takeover.unfence_node(DEV_ID, "node-a")) + self.assertEqual(self.dev(db).fenced_node_ids, []) + + +class TestTakeoverAfterGrowth(_TakeoverCase): + """A node that kept the base bdev over a LUN from before a growth + (device_controller.grow_cluster_devices) still has it at the old size; + the takeover re-reads the block device instead of failing.""" + + def _grown_bed(self): + db, dev = _bed() + dev.size = 2 * SIZE + for nid in ("node-b", "node-c", "node-d"): + db.nodes[nid].agent.lun_info.return_value = ([_lun_entry(size=2 * SIZE)], None) + b = db.nodes["node-b"] + base = utils.aio_bdev_name_for_serial(dev.serial_number) + b.spdk.bdevs[base] = {"name": base, "block_size": 4096, "num_blocks": SIZE // 4096} + return db, b, base + + def test_stale_base_bdev_is_rescanned(self): + db, b, base = self._grown_bed() + + def _rescan(name): + b.spdk.calls.append(("bdev_aio_rescan", (name,), {})) + b.spdk.bdevs[name]["num_blocks"] = 2 * SIZE // 4096 + return True + b.spdk.bdev_aio_rescan = _rescan + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(b.spdk.ops("bdev_aio_rescan"), [("bdev_aio_rescan", (base,), {})]) + b.agent.rescan_block_device.assert_called_once_with(BY_ID) + self.assertEqual(b.spdk.ops("bdev_aio_create"), []) + self.assertEqual(self.dev(db).serving_node(), "node-b") + + def test_base_bdev_still_small_after_rescan_fails_on_that_node(self): + db, b, base = self._grown_bed() + b.spdk.bdev_aio_rescan = lambda name: True + self.assertNotEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(b.spdk.ops("bdev_alceml_create"), []) + + +# ------------------------------------- fence by persistent reservation + +class TestFenceByReservation(_TakeoverCase): + """ultra #236: the takeover's alceml opens with the standby's key and + preempts the failed node's reservation (fence in ms); the array fence + follows off the I/O path. Without the reservation: array fence first.""" + + def _roles(self, node, *answers): + seq = list(answers) + node.spdk.alceml_get_role = lambda name: seq.pop(0) if len(seq) > 1 else seq[0] + + def _bed(self): + db, dev = _bed() + dev.lun_reservations = True + return db, dev + + def _wait_background_fence(self): + import time as _t + for _ in range(100): + if FakeArrayPlugin.calls: + return + _t.sleep(0.02) + + def test_reservation_fences_before_the_array(self): + db, _ = self._bed() + b = db.nodes["node-b"] + self._roles(b, {"role": "activating", "state": "act_reservation"}, + {"role": "active", "state": "active", "epoch": 2}) + order = [] + create = b.spdk.bdev_alceml_create + b.spdk.bdev_alceml_create = lambda *a, **k: (order.append(("alceml", FakeArrayPlugin.calls[:])), + create(*a, **k))[1] + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + # Nothing at the array before the alceml (whose open preempts). + self.assertEqual(order, [("alceml", [])]) + alceml = b.spdk.ops("bdev_alceml_create")[0] + self.assertEqual(alceml[2]["reservation_key"], multi_attach.reservation_key(b)) + self.assertEqual(alceml[2]["reservation_key"], 2 << 32) + self.assertNotIn("standby", alceml[2]) + self.assertEqual(self.dev(db).serving_node(), "node-b") + self.assertIn("node-a", self.dev(db).fenced_node_ids) + # The array fence follows in the background. + self._wait_background_fence() + self.assertEqual(FakeArrayPlugin.calls, [("fence", WWN, "host-node-a")]) + + def test_failed_activation_falls_back_to_the_array_fence(self): + db, _ = self._bed() + b = db.nodes["node-b"] + self._roles(b, {"role": "standby", "state": "act_failed"}, + {"role": "active", "state": "active"}) + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(FakeArrayPlugin.calls, [("fence", WWN, "host-node-a")]) + # The first instance was removed, the second one took the reservation. + self.assertEqual(len(b.spdk.ops("bdev_alceml_create")), 2) + self.assertEqual(len(b.spdk.ops("bdev_alceml_delete")), 1) + self.assertEqual(self.dev(db).serving_node(), "node-b") + + def test_dead_standby_is_left_for_the_next_one(self): + # 2026-11-20 FTT2 bed, dual reboot sn2 + hard reset sn3: the standby + # chosen was rebooting; the LUN must still be taken over by another. + db, _ = self._bed() + b, c = db.nodes["node-b"], db.nodes["node-c"] + + def b_dies(name): + b.spdk.down = True + return {"error": "Could not reach remote"} + b.spdk.alceml_get_role = b_dies + self._roles(c, {"role": "active", "state": "active", "epoch": 3}) + with patch.object(constants, "MULTI_ATTACH_ACTIVATION_WAIT_SEC", 0.2): + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).serving_node(), "node-c") + self.assertIn("node-a", self.dev(db).fenced_node_ids) + # The array fence ran (the reservation was not taken on node-b). + self.assertIn(("fence", WWN, "host-node-a"), FakeArrayPlugin.calls) + # No cleanup calls into the dead standby. + self.assertEqual(b.spdk.ops("bdev_alceml_delete"), []) + + def test_dead_standby_costs_no_activation_wait(self): + # 2026-11-20 FTT2 bed, case 7 again: the whole 5 s activation wait + # and an array fence were spent on a rebooting standby (13 s stall). + db, _ = self._bed() + b, c = db.nodes["node-b"], db.nodes["node-c"] + + def b_gone(name): + b.spdk.down = True + raise ConnectionError("Could not reach remote") + b.spdk.alceml_get_role = b_gone + self._roles(c, {"role": "active", "state": "active", "epoch": 3}) + fenced_before_c = [] + create_c = c.spdk.bdev_alceml_create + c.spdk.bdev_alceml_create = lambda *a, **k: ( + fenced_before_c.append(list(FakeArrayPlugin.calls)), create_c(*a, **k))[1] + t0 = time.monotonic() + with patch.object(constants, "MULTI_ATTACH_ACTIVATION_WAIT_SEC", 5): + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertLess(time.monotonic() - t0, 2.0) + self.assertEqual(self.dev(db).serving_node(), "node-c") + # node-c preempted by reservation: no array fence before it opened. + self.assertEqual(fenced_before_c, [[]]) + self.assertEqual(len(b.spdk.ops("bdev_alceml_create")), 1) + self._wait_background_fence() + self.assertEqual(FakeArrayPlugin.calls, [("fence", WWN, "host-node-a")]) + + def test_a_hanging_candidate_does_not_hold_the_choice(self): + # 2026-11-20 FTT2 bed, network cuts of two nodes: the first candidate's + # agent did not answer, the choice waited until the 90 s deadline. + db, _ = self._bed() + b, c = db.nodes["node-b"], db.nodes["node-c"] + gate = threading.Event() + b.agent.lun_info.side_effect = lambda *_a, **_k: (gate.wait(10), (None, "gone"))[1] + self._roles(c, {"role": "active", "state": "active", "epoch": 3}) + t0 = time.monotonic() + with patch.object(lun_takeover, "STANDBY_PROBE_TIMEOUT_SEC", 3.0): + out = self.run_takeover(db, deadline_sec=4.0) + gate.set() + self.assertEqual(out, lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).serving_node(), "node-c") + self.assertLess(time.monotonic() - t0, 2.0) + + def test_candidate_order_wins_when_several_answer(self): + db, _ = self._bed() + db.nodes["node-b"].spdk.alceml_get_role = lambda name: {"role": "active", "state": "active"} + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).serving_node(), "node-b") + + def test_no_standby_left_fails(self): + db, _ = self._bed() + def dies(spdk): + def role(name): + spdk.down = True + return {"error": "gone"} + return role + for nid in ("node-b", "node-c", "node-d"): + db.nodes[nid].spdk.alceml_get_role = dies(db.nodes[nid].spdk) + with patch.object(constants, "MULTI_ATTACH_ACTIVATION_WAIT_SEC", 0.2): + self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) + self.assertEqual(self.dev(db).serving_node(), "node-a") + + def test_without_reservations_the_array_fences_first(self): + db, _ = _bed() + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + alceml = db.nodes["node-b"].spdk.ops("bdev_alceml_create")[0] + self.assertNotIn("reservation_key", alceml[2]) + + +class TestReservationKeys(unittest.TestCase): + + def setUp(self): + array.forget_plugin() + self.addCleanup(array.forget_plugin) + + def test_key_is_the_node_block_plus_one_in_the_upper_half(self): + n = _node("n", "10.0.0.9", 0) + self.assertEqual(multi_attach.reservation_key(n), 1 << 32) + n.cntlid_block = 41 + self.assertEqual(multi_attach.reservation_key(n), 42 << 32) + n.cntlid_block = -1 + self.assertEqual(multi_attach.reservation_key(n), 0) + + def test_kwargs_follow_the_device_record(self): + db, dev = _bed() + b = db.nodes["node-b"] + self.assertNotIn("reservation_key", multi_attach.alceml_create_kwargs(dev, b, db.cluster)) + dev.lun_reservations = True + kw = multi_attach.alceml_create_kwargs(dev, b, db.cluster) + self.assertEqual(kw["reservation_key"], 2 << 32) + self.assertEqual(kw["stuck_io_timeout_ms"], constants.MULTI_ATTACH_ALCEML_STUCK_IO_MS) + dev.multi_attach = False + self.assertEqual(multi_attach.alceml_create_kwargs(dev, b, db.cluster), {}) + + def test_decided_with_the_array_capability_and_checksum(self): + class PR(FakeArrayPlugin): + def capabilities(self): + return dict(super().capabilities(), supports_reservations=True) + array.register_plugin("pr-test", PR) + self.addCleanup(array.unregister_plugin, "pr-test") + db, dev = _bed() + self.assertFalse(multi_attach.reservations_supported(db.cluster, dev)) # fake: no PR + db.cluster.array_plugin = "pr-test" + array.forget_plugin() + self.assertTrue(multi_attach.reservations_supported(db.cluster, dev)) + db.cluster.inline_checksum, dev.md_supported = True, False # method 2 + self.assertFalse(multi_attach.reservations_supported(db.cluster, dev)) + with patch.object(constants, "MULTI_ATTACH_RESERVATIONS", False): + db.cluster.inline_checksum = False + self.assertFalse(multi_attach.reservations_supported(db.cluster, dev)) diff --git a/tests/unit/test_lun_warm_standby.py b/tests/unit/test_lun_warm_standby.py new file mode 100644 index 0000000000..545b9f2941 --- /dev/null +++ b/tests/unit/test_lun_warm_standby.py @@ -0,0 +1,484 @@ +"""Multi-attach warm standby (Cluster.multi_attach_warm_standby, opt-in). + + - lun_takeover.ensure_warm_standbys: a standby alceml (standby=true, + open-only, the reservation key) on attached non-serving nodes only; never + exported; never on fenced / serving / unattached nodes or on clusters + without the flag; closed when the node is no longer a standby; refused + beyond the node's memory budget, with a cluster event. + - the takeover reuses a warm standby: no alceml create, the export is + built, the activation is awaited after the path switch; a standby that + does not activate falls back to the array fence and a cold open. + - memory: warm-standby LUNs count in the node's restart budget. + - plumbing: cluster flag, monitor driver. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import ( + cluster_ops, + constants, + multi_attach, + storage_node_ops, + utils, +) +from simplyblock_core.array.fake import FakeArrayPlugin +from simplyblock_core.controllers import device_controller, lun_takeover +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import storage_node_monitor +from tests.unit.test_lun_takeover import BY_ID, DEV_ID, SIZE, WWN, _bed, _TakeoverCase + +ALCEML = f"alceml_{DEV_ID}" +BASE = utils.aio_bdev_name_for_serial(WWN) + + +def _warm_bed(flag=True, reservations=True): + """_bed with the owner node-a ONLINE and serving its LUN; node-b / node-c + its LVS secondary / tertiary (attached standbys), node-d a peer.""" + db, dev = _bed() + db.cluster.multi_attach_warm_standby = flag + dev.lun_reservations = reservations + db.nodes["node-a"].status = StorageNode.STATUS_ONLINE + return db, dev + + +def _role(spdk, fn): + spdk.alceml_get_role = fn + + +def _standby_role(_name): + return {"role": "standby", "state": "standby", "standby_at_open": True, "warm": True, + "refreshes_count": 7} + + +class _WarmCase(_TakeoverCase): + + def ensure(self, db, nid): + self.m_DBController.return_value = db + return lun_takeover.ensure_warm_standbys(db.nodes[nid]) + + +# ------------------------------------------------------ keeping standbys + +class TestEnsureWarmStandbys(_WarmCase): + + def test_created_on_attached_non_serving_nodes_only(self): + db, _ = _warm_bed() + for nid in ("node-a", "node-b", "node-c", "node-d"): + self.ensure(db, nid) + for nid in ("node-b", "node-c"): + spdk = db.nodes[nid].spdk + calls = spdk.ops("bdev_alceml_create") + self.assertEqual(len(calls), 1, nid) + kw = calls[0][2] + self.assertEqual(calls[0][1][:3], (ALCEML, BASE, DEV_ID)) + self.assertIs(kw["standby"], True) + self.assertEqual(kw["pba_init_mode"], lun_takeover.ALCEML_OPEN_ONLY) + self.assertEqual(kw["reservation_key"], multi_attach.reservation_key(db.nodes[nid])) + self.assertEqual(kw["refresh_interval_ms"], constants.MULTI_ATTACH_WARM_REFRESH_MS) + self.assertEqual(kw["stuck_io_timeout_ms"], constants.MULTI_ATTACH_ALCEML_STUCK_IO_MS) + # Over the node's own path to the LUN (its agent's lun_info). + self.assertEqual(spdk.ops("bdev_aio_create")[0][1][:2], (BASE, BY_ID)) + # Never exported: a peer's I/O would activate it. + self.assertEqual(spdk.ops("bdev_PT_NoExcl_create"), []) + self.assertEqual(spdk.ops("subsystem_create"), []) + self.assertEqual(spdk.ops("nvmf_subsystem_add_ns"), []) + # The serving node and an unattached peer get nothing. + self.assertEqual(db.nodes["node-a"].spdk.calls, []) + self.assertEqual(db.nodes["node-d"].spdk.calls, []) + db.nodes["node-d"].agent.lun_info.assert_not_called() + + def test_idempotent(self): + db, _ = _warm_bed() + self.assertEqual(self.ensure(db, "node-b")["created"], [DEV_ID]) + _role(db.nodes["node-b"].spdk, _standby_role) + self.assertEqual(self.ensure(db, "node-b"), {"created": [], "removed": [], "over_budget": []}) + self.assertEqual(len(db.nodes["node-b"].spdk.ops("bdev_alceml_create")), 1) + + def test_not_on_a_fenced_node(self): + db, dev = _warm_bed() + dev.fenced_node_ids = ["node-c"] + self.ensure(db, "node-c") + self.assertEqual(db.nodes["node-c"].spdk.calls, []) + + def test_not_on_the_serving_node_after_a_takeover(self): + db, dev = _warm_bed() + dev.serving_node_id = "node-b" + dev.fenced_node_ids = ["node-a"] + self.ensure(db, "node-b") + self.assertEqual(db.nodes["node-b"].spdk.ops("bdev_alceml_create"), []) + # The owner, fenced, gets none either; the tertiary does. + self.ensure(db, "node-a") + self.assertEqual(db.nodes["node-a"].spdk.ops("bdev_alceml_create"), []) + self.ensure(db, "node-c") + self.assertEqual(len(db.nodes["node-c"].spdk.ops("bdev_alceml_create")), 1) + + def test_not_on_offline_node_or_device(self): + db, dev = _warm_bed() + db.nodes["node-b"].status = StorageNode.STATUS_OFFLINE + self.ensure(db, "node-b") + self.assertEqual(db.nodes["node-b"].spdk.calls, []) + db.nodes["node-b"].status = StorageNode.STATUS_ONLINE + dev.status = "unavailable" + self.ensure(db, "node-b") + self.assertEqual(db.nodes["node-b"].spdk.ops("bdev_alceml_create"), []) + + def test_not_without_the_flag_or_reservations(self): + for flag, res in ((False, True), (True, False)): + with self.subTest(flag=flag, reservations=res): + db, _ = _warm_bed(flag=flag, reservations=res) + for nid in db.nodes: + self.ensure(db, nid) + self.assertEqual(db.nodes[nid].spdk.calls, []) + db.nodes[nid].agent.lun_info.assert_not_called() + + def test_not_while_a_takeover_of_the_lun_runs(self): + db, _ = _warm_bed() + lock = lun_takeover._local_lock(DEV_ID) + with lock: + self.ensure(db, "node-b") + self.assertEqual(db.nodes["node-b"].spdk.ops("bdev_alceml_create"), []) + + def test_lun_not_visible_opens_nothing(self): + db, _ = _warm_bed() + db.nodes["node-b"].agent.lun_info.return_value = ([], None) + self.assertEqual(self.ensure(db, "node-b")["created"], []) + self.assertEqual(db.nodes["node-b"].spdk.calls, []) + + def test_failed_open_closes_the_base_bdev(self): + db, _ = _warm_bed() + b = db.nodes["node-b"] + b.spdk.fail.add("bdev_alceml_create") + self.assertEqual(self.ensure(db, "node-b")["created"], []) + self.assertNotIn(BASE, b.spdk.bdevs) + self.assertTrue(b.spdk.ops("bdev_aio_delete")) + + def test_removed_standby_role_cleans_up(self): + db, dev = _warm_bed() + c = db.nodes["node-c"] + self.ensure(db, "node-c") + self.assertIn(ALCEML, c.spdk.bdevs) + _role(c.spdk, _standby_role) + # node-c is no longer the owner's tertiary: not attached any more. + db.nodes["node-a"].tertiary_node_id = "" + report = self.ensure(db, "node-c") + self.assertEqual(report["removed"], [DEV_ID]) + self.assertNotIn(ALCEML, c.spdk.bdevs) + self.assertNotIn(BASE, c.spdk.bdevs) + deletes = [op for op, *_ in c.spdk.calls if op.endswith("_delete")] + self.assertEqual(deletes, ["bdev_alceml_delete", "bdev_aio_delete"]) + + def test_fenced_standby_is_closed(self): + db, dev = _warm_bed() + b = db.nodes["node-b"] + self.ensure(db, "node-b") + _role(b.spdk, _standby_role) + dev.fenced_node_ids = ["node-b"] + self.assertEqual(self.ensure(db, "node-b")["removed"], [DEV_ID]) + self.assertNotIn(ALCEML, b.spdk.bdevs) + + def test_exported_or_active_stack_is_never_closed(self): + db, dev = _warm_bed() + b = db.nodes["node-b"] + dev.fenced_node_ids = ["node-b"] + b.spdk.bdevs[ALCEML] = {"name": ALCEML} + _role(b.spdk, lambda _n: {"role": "active", "state": "active"}) + self.assertEqual(self.ensure(db, "node-b")["removed"], []) + _role(b.spdk, _standby_role) + b.spdk.bdevs[f"{ALCEML}_PT"] = {"name": f"{ALCEML}_PT"} + self.assertEqual(self.ensure(db, "node-b")["removed"], []) + self.assertIn(ALCEML, b.spdk.bdevs) + + def test_owner_standby_keeps_its_base_bdev(self): + # The owner, no longer serving and not a standby: its alceml closes, + # its own base bdev stays (its restart path owns it). + db, dev = _warm_bed() + a = db.nodes["node-a"] + dev.serving_node_id = "node-b" + dev.fenced_node_ids = ["node-a"] + a.spdk.bdevs[BASE] = {"name": BASE} + a.spdk.bdevs[ALCEML] = {"name": ALCEML} + _role(a.spdk, _standby_role) + self.assertEqual(self.ensure(db, "node-a")["removed"], [DEV_ID]) + self.assertNotIn(ALCEML, a.spdk.bdevs) + self.assertIn(BASE, a.spdk.bdevs) + + +class TestRetireKeepsWarmStandby(_WarmCase): + """retire_stale_local_stack (node back ONLINE without a restart) keeps + the warm standby of a LUN served elsewhere; a fenced node's is retired.""" + + def _owner_standby(self): + db, dev = _warm_bed() + a = db.nodes["node-a"] + dev.serving_node_id = "node-b" + a.spdk.bdevs[ALCEML] = {"name": ALCEML} + _role(a.spdk, _standby_role) + p = patch("simplyblock_core.db_controller.DBController", return_value=db) + p.start() + self.addCleanup(p.stop) + return db, dev, a + + def test_kept(self): + _db, _dev, a = self._owner_standby() + self.assertEqual(lun_takeover.retire_stale_local_stack(a), []) + self.assertIn(ALCEML, a.spdk.bdevs) + + def test_retired_when_fenced_or_deposed(self): + _db, dev, a = self._owner_standby() + dev.fenced_node_ids = ["node-a"] + self.assertEqual(lun_takeover.retire_stale_local_stack(a), [DEV_ID]) + self.assertNotIn(ALCEML, a.spdk.bdevs) + # A deposed former active (not opened standby) is stale as well. + _db, dev, a = self._owner_standby() + _role(a.spdk, lambda _n: {"role": "standby", "state": "deposed", "standby_at_open": False}) + self.assertEqual(lun_takeover.retire_stale_local_stack(a), [DEV_ID]) + + +# --------------------------------------------------------------- memory + +class TestWarmStandbyMemory(_WarmCase): + + def test_over_budget_is_refused_with_one_event(self): + db, _ = _warm_bed() + b = db.nodes["node-b"] + b.minimum_sys_memory = utils.sys_memory_for_capacity(0) # no room for a page map + report = self.ensure(db, "node-b") + self.assertEqual(report["over_budget"], [DEV_ID]) + self.assertEqual(b.spdk.ops("bdev_alceml_create"), []) + budget_events = [e for e in self.events if "memory budget" in e["message"]] + self.assertEqual(len(budget_events), 1) + self.assertEqual(budget_events[0]["node_id"], "node-b") + # Not repeated while nothing changes. + self.ensure(db, "node-b") + self.assertEqual(len([e for e in self.events if "memory budget" in e["message"]]), 1) + + def test_within_budget_is_opened(self): + db, _ = _warm_bed() + b = db.nodes["node-b"] + b.minimum_sys_memory = utils.sys_memory_for_capacity(SIZE) + self.assertEqual(self.ensure(db, "node-b")["created"], [DEV_ID]) + self.assertFalse([e for e in self.events if "memory budget" in e["message"]]) + + def test_restart_budget_counts_warm_standby_luns(self): + db, dev = _warm_bed() + b, d = db.nodes["node-b"], db.nodes["node-d"] + self.assertEqual(multi_attach.warm_standby_capacity(db.cluster, "node-b", list(db.nodes.values())), + SIZE) + self.assertEqual(storage_node_ops._warm_standby_capacity(db, b), SIZE) + # A device unavailable for a while still counts: its standby returns. + dev.status = "unavailable" + self.assertEqual(storage_node_ops._warm_standby_capacity(db, b), SIZE) + # Not a standby (unattached peer, serving node), not a warm cluster. + self.assertEqual(storage_node_ops._warm_standby_capacity(db, d), 0) + self.assertEqual(storage_node_ops._warm_standby_capacity(db, db.nodes["node-a"]), 0) + db.cluster.multi_attach_warm_standby = False + self.assertEqual(storage_node_ops._warm_standby_capacity(db, b), 0) + + +# ------------------------------------------------------------- takeover + +class TestWarmTakeover(_WarmCase): + + def setUp(self): + super().setUp() + p = patch.object(device_controller, "device_set_unavailable", return_value=True) + self.m_unavailable = p.start() + self.addCleanup(p.stop) + + def _with_standby(self): + """The secondary holds a warm standby (opened by ensure_warm_standbys); + then the serving owner fails.""" + db, dev = _warm_bed() + self.ensure(db, "node-b") + b = db.nodes["node-b"] + self.assertIn(ALCEML, b.spdk.bdevs) + b.spdk.calls.clear() + db.nodes["node-a"].status = StorageNode.STATUS_OFFLINE + return db, dev, b + + def test_reuses_the_warm_standby(self): + db, _, b = self._with_standby() + seen = [] + + def role(_name): + # The peers' I/O reaches it once the paths are switched. + switched = bool(b.spdk.ops("attach")) + seen.append(switched) + if switched: + return {"role": "active", "state": "active", "epoch": 3} + return _standby_role(_name) + _role(b.spdk, role) + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(b.spdk.ops("bdev_alceml_create"), []) + self.assertEqual(b.spdk.ops("bdev_alceml_delete"), []) + self.assertEqual(len(b.spdk.ops("bdev_PT_NoExcl_create")), 1) + self.assertEqual(len(b.spdk.ops("subsystem_create")), 1) + self.assertEqual(len(b.spdk.ops("nvmf_subsystem_add_ns")), 1) + # Checked once before the export (standby), then awaited after the + # path switch until active. + self.assertEqual(seen[0], False) + self.assertEqual(seen[-1], True) + dev = self.dev(db) + self.assertEqual(dev.serving_node(), "node-b") + self.assertIn("node-a", dev.fenced_node_ids) + self.m_unavailable.assert_not_called() + + def test_activation_failure_falls_back_to_a_cold_open(self): + db, _, b = self._with_standby() + + def role(_name): + # The standby never activates; the instance reopened active does. + if b.spdk.ops("bdev_alceml_create"): + return {"role": "active", "state": "active"} + return _standby_role(_name) + _role(b.spdk, role) + with patch.object(constants, "MULTI_ATTACH_ACTIVATION_WAIT_SEC", 0.2): + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + # The failed node is fenced at the array (synchronously). + self.assertIn(("fence", WWN, "host-node-a"), FakeArrayPlugin.calls) + # Export torn down, standby closed, then reopened active with the key. + ops = [op for op, *_ in b.spdk.calls] + self.assertLess(ops.index("attach"), ops.index("subsystem_delete")) + self.assertLess(ops.index("bdev_alceml_delete"), ops.index("bdev_alceml_create")) + create = b.spdk.ops("bdev_alceml_create") + self.assertEqual(len(create), 1) + self.assertNotIn("standby", create[0][2]) + self.assertEqual(create[0][2]["reservation_key"], multi_attach.reservation_key(b)) + self.assertEqual(create[0][2]["pba_init_mode"], lun_takeover.ALCEML_OPEN_ONLY) + self.assertEqual(len(b.spdk.ops("subsystem_create")), 2) + self.assertIn(BASE, b.spdk.bdevs) + dev = self.dev(db) + self.assertEqual(dev.serving_node(), "node-b") + self.assertIn("node-a", dev.fenced_node_ids) + self.assertTrue(any("did not activate" in e["message"] for e in self.events)) + self.m_unavailable.assert_not_called() + + def test_cold_fallback_failure_sets_the_device_unavailable(self): + db, _, b = self._with_standby() + _role(b.spdk, _standby_role) + b.spdk.fail.add("bdev_alceml_create") + with patch.object(constants, "MULTI_ATTACH_ACTIVATION_WAIT_SEC", 0.2): + self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) + self.m_unavailable.assert_called_once_with(DEV_ID) + self.assertTrue(any("cold open failed" in e["message"] for e in self.events)) + + def test_standby_whose_activation_failed_is_reopened(self): + db, _, b = self._with_standby() + + def role(_name): + if b.spdk.ops("bdev_alceml_create"): + return {"role": "active", "state": "active"} + return {"role": "standby", "state": "act_failed"} + _role(b.spdk, role) + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(len(b.spdk.ops("bdev_alceml_delete")), 1) + create = b.spdk.ops("bdev_alceml_create") + self.assertEqual(len(create), 1) + self.assertNotIn("standby", create[0][2]) + + def test_unresponsive_warm_standby_moves_to_the_next(self): + db, _, b = self._with_standby() + c = db.nodes["node-c"] + + def b_dies(_name): + b.spdk.down = True + raise ConnectionError("gone") + _role(b.spdk, b_dies) + _role(c.spdk, lambda _n: {"role": "active", "state": "active"}) + with patch.object(constants, "MULTI_ATTACH_ACTIVATION_WAIT_SEC", 0.2): + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).serving_node(), "node-c") + + def test_without_the_flag_an_existing_standby_is_not_reused(self): + # Flag off: the regular path (wait for the reservation before the export). + db, _, b = self._with_standby() + db.cluster.multi_attach_warm_standby = False + _role(b.spdk, lambda _n: {"role": "active", "state": "active"}) + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(b.spdk.ops("bdev_alceml_create"), []) + self.assertEqual(self.dev(db).serving_node(), "node-b") + + +# ------------------------------------------------------------- plumbing + +class TestWarmStandbyPlumbing(unittest.TestCase): + + def test_helper(self): + db, dev = _warm_bed() + self.assertTrue(multi_attach.warm_standby_enabled(db.cluster, dev)) + db.cluster.multi_attach_warm_standby = False + self.assertFalse(multi_attach.warm_standby_enabled(db.cluster, dev)) + db.cluster.multi_attach_warm_standby = True + dev.lun_reservations = False + self.assertFalse(multi_attach.warm_standby_enabled(db.cluster, dev)) + self.assertFalse(multi_attach.warm_standby_enabled(Cluster(), dev)) + + def test_cluster_flag_needs_multi_attach(self): + self.assertFalse(Cluster().multi_attach_warm_standby) + with patch.object(constants, "MULTI_ATTACH_WARM_STANDBY_ALLOWED", True): + with self.assertRaises(ValueError): + cluster_ops._validate_warm_standby(False, True) + cluster_ops._validate_warm_standby(True, True) + cluster_ops._validate_warm_standby(False, False) + + def test_refused_while_disabled(self): + # 2026-11-20 bed: stale data after a warm takeover. + with patch.object(constants, "MULTI_ATTACH_WARM_STANDBY_ALLOWED", False): + with self.assertRaises(ValueError) as cm: + cluster_ops._validate_warm_standby(True, True) + self.assertIn("disabled", str(cm.exception)) + + def test_cli_flag(self): + import argparse + + from simplyblock_cli.cli import CLIWrapper + cli = CLIWrapper.__new__(CLIWrapper) + cli.developer_mode = False + for cmd in ("create", "add"): + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="c") + with patch.object(CLIWrapper, "add_sub_command", + lambda self, subp, name, help, **k: subp.add_parser(name)): + getattr(cli, f"init_cluster__{cmd}")(sub) + args, _ = parser.parse_known_args([cmd, "--multi-attach-warm-standby"]) + self.assertTrue(args.multi_attach_warm_standby, cmd) + + def test_rpc_sends_refresh_interval_for_a_standby_only(self): + from simplyblock_core.rpc_client import RPCClient + rpc = RPCClient.__new__(RPCClient) + rpc._request = MagicMock(return_value=True) + rpc.bdev_alceml_create("a", "b", "u", pba_init_mode=1, reservation_key=1 << 32, + standby=True, refresh_interval_ms=2000) + params = rpc._request.call_args.kwargs + self.assertEqual((params["standby"], params["refresh_interval_ms"]), (True, 2000)) + rpc.bdev_alceml_create("a", "b", "u", pba_init_mode=1, refresh_interval_ms=2000) + params = rpc._request.call_args.kwargs + self.assertNotIn("standby", params) + self.assertNotIn("refresh_interval_ms", params) + + def test_monitor_drives_warm_clusters_only(self): + db, _ = _warm_bed() + node = db.nodes["node-b"] + storage_node_monitor._warm_standby_last.clear() + storage_node_monitor._warm_standby_threads.clear() + with patch.object(storage_node_monitor, "db") as mdb, \ + patch.object(lun_takeover, "ensure_warm_standbys") as ensure: + mdb.get_cluster_by_id.return_value = db.cluster + storage_node_monitor._ensure_warm_standbys(node) + storage_node_monitor._warm_standby_threads["node-b"].join(5) + ensure.assert_called_once_with(node) + # Throttled per node. + storage_node_monitor._ensure_warm_standbys(node) + self.assertEqual(ensure.call_count, 1) + storage_node_monitor._warm_standby_last.clear() + db.cluster.multi_attach_warm_standby = False + storage_node_monitor._ensure_warm_standbys(node) + self.assertEqual(ensure.call_count, 1) + storage_node_monitor._warm_standby_last.clear() + storage_node_monitor._warm_standby_threads.clear() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_lun_watchdog.py b/tests/unit/test_lun_watchdog.py new file mode 100644 index 0000000000..5bf01580ee --- /dev/null +++ b/tests/unit/test_lun_watchdog.py @@ -0,0 +1,285 @@ +"""services/lun_watchdog: fast loss detection of nodes serving multi-attach +LUNs, and the immediate takeover of their LUNs. + + - probe: SPDK RPC first; the node agent decides between a killed SPDK, a + slow one and a node that is gone. + - decisions: SPDK down -> lost at once; no answer -> lost after + MULTI_ATTACH_LOST_AFTER_SEC; SPDK running -> never; an answer resets. + - fire: every LUN the node serves, rate-limited; never on a regular + cluster; never while this process may be the one cut off. + - watched: multi-attach clusters, nodes that serve a LUN. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import array, constants +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import lun_watchdog as wd +from tests.unit.test_lun_takeover import DEV_ID, _bed + +LOST = constants.MULTI_ATTACH_LOST_AFTER_SEC + + +class _Db: + """FakeDB of test_lun_takeover with get_clusters.""" + + def __init__(self, db): + self.inner = db + + def __getattr__(self, name): + return getattr(self.inner, name) + + def get_clusters(self): + return [self.inner.cluster] + + +def _setup(serving_a=True): + """Bed of test_lun_takeover with node-a ONLINE and serving its LUN.""" + db, dev = _bed() + a = db.nodes["node-a"] + a.status = StorageNode.STATUS_ONLINE + db.cluster.max_fault_tolerance = 1 + if not serving_a: + dev.serving_node_id = "node-b" + return _Db(db), dev + + +class TestProbe(unittest.TestCase): + + def _node(self, rpc_ok=True, agent=(True, None), agent_raises=False): + node = MagicMock() + node.rpc_port, node.cluster_id = 5260, "cluster-1" + rpc = MagicMock() + if not rpc_ok: + rpc.get_version.side_effect = ConnectionError("refused") + node.rpc_client.return_value = rpc + api = MagicMock() + if agent_raises: + api.spdk_process_is_up.side_effect = RuntimeError("timeout") + else: + api.spdk_process_is_up.return_value = agent + node.client.return_value = api + return node + + def test_rpc_answers(self): + node = self._node() + self.assertEqual(wd.probe(node), wd.OK) + node.client.assert_not_called() + kw = node.rpc_client.call_args.kwargs + self.assertEqual((kw["timeout"], kw["retry"]), (constants.MULTI_ATTACH_PROBE_TIMEOUT_SEC, 0)) + + def test_spdk_killed(self): + self.assertEqual(wd.probe(self._node(False, (False, "socket down"))), wd.SPDK_DOWN) + + def test_spdk_slow(self): + self.assertEqual(wd.probe(self._node(False, (True, None))), wd.SPDK_UP) + + def test_node_gone(self): + node = self._node(False, agent_raises=True) + self.assertEqual(wd.probe(node), wd.NO_ANSWER) + kw = node.client.call_args.kwargs + self.assertEqual((kw["retry"], kw["connect_retry"]), (0, 0)) + + def test_agent_without_verdict(self): + self.assertEqual(wd.probe(self._node(False, (None, "error"))), wd.NO_ANSWER) + + def test_failing_asks_both_at_once(self): + node = self._node(False, agent_raises=True) + self.assertEqual(wd.probe(node, failing=True), wd.NO_ANSWER) + node.client.assert_called_once() + self.assertEqual(wd.probe(self._node(True), failing=True), wd.OK) + + +class TestDecisions(unittest.TestCase): + + def setUp(self): + self.db, self.dev = _setup() + self.w = wd.Watchdog(MagicMock(), db=self.db) + self.a = self.db.nodes["node-a"] + + def test_spdk_down_is_lost_at_once(self): + self.assertEqual(self.w.record(self.a, wd.SPDK_DOWN, 100.0), "SPDK not running") + + def test_no_answer_is_lost_after_the_window(self): + # Probes of a node that does not answer take PROBE_TIMEOUT each. + t = 100.0 + step = constants.MULTI_ATTACH_PROBE_TIMEOUT_SEC + reasons = [] + while t < 100.0 + LOST + step: + reasons.append(self.w.record(self.a, wd.NO_ANSWER, t)) + t += step + self.assertTrue(all(r is None for r in reasons[:-1]), reasons) + self.assertTrue(reasons[-1].startswith("no answer for")) + # Detection inside the 9 s stall budget with room for the takeover. + self.assertLessEqual(LOST + 2 * step, 4.0) + + def test_silence_counts_from_the_first_unanswered_probe(self): + # The first probe was sent at 100 and gave up (RPC + agent timeouts) + # at 101.6; the next one, asking both at once, ends at 102.4. + self.assertIsNone(self.w.record(self.a, wd.NO_ANSWER, 101.6, sent=100.0)) + self.assertTrue(self.w.record(self.a, wd.NO_ANSWER, 102.4, sent=101.6) + .startswith("no answer for 2.4s")) + + def test_running_spdk_is_never_lost(self): + for t in range(60): + self.assertIsNone(self.w.record(self.a, wd.SPDK_UP, 100.0 + t)) + + def test_spdk_running_restarts_the_silence(self): + # Slow RPC while the agent sees SPDK running, then one agent timeout: + # not "no answer for" the whole stretch. + self.assertIsNone(self.w.record(self.a, wd.NO_ANSWER, 100.0)) + for t in (101.0, 102.0, 103.0, 104.0, 105.0): + self.assertIsNone(self.w.record(self.a, wd.SPDK_UP, t)) + self.assertIsNone(self.w.record(self.a, wd.NO_ANSWER, 106.0, sent=105.2)) + self.assertTrue(self.w.record(self.a, wd.NO_ANSWER, 107.4, sent=106.0) + .startswith("no answer for 2.2s")) + + def test_slow_node_is_not_counted_as_failing(self): + self.w.record(self.a, wd.SPDK_UP, 100.0) + self.assertIsNone(self.w.tracks["node-a"].first_fail) + + def test_an_answer_within_the_window_cancels(self): + self.assertIsNone(self.w.record(self.a, wd.NO_ANSWER, 100.0)) + self.assertIsNone(self.w.record(self.a, wd.OK, 100.0 + LOST - 0.5)) + self.assertIsNone(self.w.record(self.a, wd.NO_ANSWER, 100.0 + LOST + 0.1)) + + +class TestFire(unittest.TestCase): + + def setUp(self): + self.db, self.dev = _setup() + self.start = MagicMock() + self.w = wd.Watchdog(self.start, db=self.db) + # Every node probed fine at t=100. + for n in self.db.nodes.values(): + self.w.record(n, wd.OK, 100.0) + + def _lose(self, nid, t=101.0): + reason = self.w.record(self.db.nodes[nid], wd.SPDK_DOWN, t) + return self.w.fire(nid, reason, t) + + def test_takes_over_every_lun_the_node_serves(self): + self.assertEqual(self._lose("node-a"), [DEV_ID]) + self.start.assert_called_once_with(DEV_ID, "node-a", "node lost: SPDK not running") + + def test_rate_limited(self): + self._lose("node-a", 101.0) + self._lose("node-a", 102.0) + self.assertEqual(self.start.call_count, 1) + for nid in ("node-b", "node-c", "node-d"): + self.w.record(self.db.nodes[nid], wd.OK, 101.0 + wd.REFIRE_SEC) + self._lose("node-a", 101.0 + wd.REFIRE_SEC) + self.assertEqual(self.start.call_count, 2) + + def test_lun_served_elsewhere_is_not_touched(self): + self.dev.serving_node_id = "node-b" + self.assertEqual(self._lose("node-a"), []) + self.start.assert_not_called() + + def test_failed_device_is_not_taken_over(self): + self.dev.status = NVMeDevice.STATUS_UNAVAILABLE + self.assertEqual(self._lose("node-a"), []) + + def test_regular_cluster_never(self): + self.db.cluster.multi_attach = False + self.assertEqual(self._lose("node-a"), []) + + def test_node_in_restart_is_left_to_the_restart(self): + self.db.nodes["node-a"].status = StorageNode.STATUS_RESTARTING + self.assertEqual(self._lose("node-a"), []) + + def test_not_when_more_nodes_fail_than_tolerated(self): + # This monitor lost its own network: every node looks gone. + self.w.record(self.db.nodes["node-b"], wd.NO_ANSWER, 100.5) + self.assertEqual(self._lose("node-a"), []) + self.start.assert_not_called() + + def test_two_lost_nodes_with_ftt2(self): + self.db.cluster.max_fault_tolerance = 2 + self.w.record(self.db.nodes["node-b"], wd.NO_ANSWER, 100.5) + self.assertEqual(self._lose("node-a"), [DEV_ID]) + + def test_not_when_no_other_node_answered_recently(self): + self.assertEqual(self._lose("node-a", 100.0 + LOST + 10), []) + self.start.assert_not_called() + + +class TestWatched(unittest.TestCase): + + def test_nodes_serving_a_lun(self): + db, _ = _setup() + w = wd.Watchdog(MagicMock(), db=db) + with patch.object(array, "cached_plugin_for_cluster"): + self.assertEqual(sorted(w.watched()), ["node-a"]) + + def test_takeover_moves_the_watch(self): + db, _ = _setup(serving_a=False) + w = wd.Watchdog(MagicMock(), db=db) + with patch.object(array, "cached_plugin_for_cluster"): + self.assertEqual(sorted(w.watched()), ["node-b"]) + + def test_regular_and_activating_clusters_are_not_watched(self): + for change in ("regular", "activation"): + with self.subTest(change=change): + db, _ = _setup() + if change == "regular": + db.cluster.multi_attach = False + else: + db.cluster.status = Cluster.STATUS_IN_ACTIVATION + w = wd.Watchdog(MagicMock(), db=db) + self.assertEqual(w.watched(), {}) + + def test_offline_node_is_not_watched(self): + db, _ = _setup() + db.nodes["node-a"].status = StorageNode.STATUS_OFFLINE + w = wd.Watchdog(MagicMock(), db=db) + with patch.object(array, "cached_plugin_for_cluster"): + self.assertEqual(w.watched(), {}) + + def test_array_connection_is_warmed(self): + db, _ = _setup() + db.cluster.array_plugin = "fake-test" + w = wd.Watchdog(MagicMock(), db=db) + with patch.object(array, "cached_plugin_for_cluster") as conn, \ + patch.object(wd.threading, "Thread") as thread: + thread.side_effect = lambda target, **_k: MagicMock(start=target) + w.watched() + w.watched() + conn.assert_called_once() + self.assertEqual(conn.call_args.kwargs["max_age"], constants.ARRAY_PLUGIN_CACHE_SEC / 2) + + +class TestPluginCache(unittest.TestCase): + + def setUp(self): + from simplyblock_core.array.fake import FakeArrayPlugin + self.cls = FakeArrayPlugin + FakeArrayPlugin.reset() + array.register_plugin("fake-cache", FakeArrayPlugin) + self.addCleanup(array.unregister_plugin, "fake-cache") + array.forget_plugin() + self.addCleanup(array.forget_plugin) + self.cluster = Cluster() + self.cluster.uuid = "cluster-c" + self.cluster.array_plugin = "fake-cache" + + def test_reused_until_the_configuration_changes(self): + p1 = array.cached_plugin_for_cluster(self.cluster) + self.assertIs(array.cached_plugin_for_cluster(self.cluster), p1) + self.cluster.array_options = {"x": 1} + self.assertIsNot(array.cached_plugin_for_cluster(self.cluster), p1) + self.assertEqual(len(self.cls.connected), 2) + + def test_expiry_and_forget(self): + p1 = array.cached_plugin_for_cluster(self.cluster) + self.assertIsNot(array.cached_plugin_for_cluster(self.cluster, max_age=0), p1) + p2 = array.cached_plugin_for_cluster(self.cluster) + array.forget_plugin("cluster-c") + self.assertIsNot(array.cached_plugin_for_cluster(self.cluster), p2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_migration_relaxed_mode.py b/tests/unit/test_migration_relaxed_mode.py index 44a3e72b7d..3999a94861 100644 --- a/tests/unit/test_migration_relaxed_mode.py +++ b/tests/unit/test_migration_relaxed_mode.py @@ -28,10 +28,17 @@ def test_fewer_nodes_than_chunks_is_relaxed(self): self.assertTrue(mig.needs_relaxed_placement(5, 4, 2)) self.assertTrue(mig.needs_relaxed_placement(3, 2, 2)) + def test_exactly_as_many_nodes_as_chunks_is_relaxed(self): + """Every node holds a chunk of every stripe: a strict migration away + from one of them has nowhere to go (the migration dead end).""" + self.assertTrue(mig.needs_relaxed_placement(3, 2, 1)) + self.assertTrue(mig.needs_relaxed_placement(6, 4, 2)) + self.assertTrue(mig.needs_relaxed_placement(4, 2, 2)) + def test_enough_nodes_stay_strict(self): self.assertFalse(mig.needs_relaxed_placement(3, 1, 1)) - self.assertFalse(mig.needs_relaxed_placement(3, 2, 1)) - self.assertFalse(mig.needs_relaxed_placement(6, 4, 2)) + self.assertFalse(mig.needs_relaxed_placement(4, 2, 1)) + self.assertFalse(mig.needs_relaxed_placement(7, 4, 2)) class TestMigrationRelaxedLookup(unittest.TestCase): @@ -51,7 +58,7 @@ def test_removed_nodes_do_not_count(self): def test_offline_member_still_counts(self): cluster = mock.Mock(distr_ndcs=2, distr_npcs=1) nodes = self._nodes(StorageNode.STATUS_ONLINE, StorageNode.STATUS_ONLINE, - StorageNode.STATUS_OFFLINE) + StorageNode.STATUS_ONLINE, StorageNode.STATUS_OFFLINE) with mock.patch.object(mig, "db") as db: db.get_cluster_by_id.return_value = cluster db.get_storage_nodes_by_cluster_id.return_value = nodes @@ -79,6 +86,37 @@ def test_expansion_start_sends_flag_only_when_relaxed(self): c.distr_migration_expansion_start("d") self.assertNotIn("relaxed_mode", c._request.call_args.kwargs) + def test_affinity_floor_is_gated_by_the_constant(self): + """Older data planes reject unknown parameters: affinity_floor and + allow_device_colocation go out only once DISTR_AFFINITY_FLOOR_RPC is on.""" + c = self._client() + c.bdev_get = mock.Mock(return_value=None) + with mock.patch.object(rpc_mod.constants, "DISTR_AFFINITY_FLOOR_RPC", False): + c.distr_migration_failure_start("d", 3, affinity_floor="relaxed") + self.assertNotIn("affinity_floor", c._request.call_args.kwargs) + c.distr_migration_expansion_start("d", affinity_floor="device") + self.assertNotIn("affinity_floor", c._request.call_args.kwargs) + c.bdev_distrib_create("d", 1, 2, 1, 10, 4096, ["jm"], 4096, allow_device_colocation=True) + self.assertNotIn("allow_device_colocation", c._request.call_args.kwargs) + with mock.patch.object(rpc_mod.constants, "DISTR_AFFINITY_FLOOR_RPC", True): + c.distr_migration_failure_start("d", 3, affinity_floor="relaxed") + self.assertEqual(c._request.call_args.kwargs["affinity_floor"], "relaxed") + c.distr_migration_expansion_start("d", affinity_floor="device") + self.assertEqual(c._request.call_args.kwargs["affinity_floor"], "device") + c.distr_migration_expansion_start("d") + self.assertNotIn("affinity_floor", c._request.call_args.kwargs) + c.bdev_distrib_create("d", 1, 2, 1, 10, 4096, ["jm"], 4096, allow_device_colocation=True) + self.assertIs(c._request.call_args.kwargs["allow_device_colocation"], True) + c.bdev_distrib_create("d", 1, 2, 1, 10, 4096, ["jm"], 4096) + self.assertNotIn("allow_device_colocation", c._request.call_args.kwargs) + + def test_affinity_floor_follows_raid_devices(self): + self.assertEqual(mig.migration_affinity_floor(mock.Mock(raid_devices=True)), "relaxed") + self.assertEqual(mig.migration_affinity_floor(mock.Mock(raid_devices=False)), "device") + with mock.patch.object(mig, "db") as db: + db.get_cluster_by_id.return_value = mock.Mock(raid_devices=True) + self.assertEqual(mig.migration_affinity_floor("c1"), "relaxed") + class TestRunnersPassRelaxedMode(unittest.TestCase): """Every migration runner hands the cluster's relaxed decision to the RPC.""" @@ -94,6 +132,7 @@ def test_each_runner_passes_relaxed_mode(self): for mod in (tasks_runner_failed_migration, tasks_runner_migration, tasks_runner_new_dev_migration): src = inspect.getsource(mod) self.assertIn("relaxed_mode=mig.migration_relaxed(", src, mod.__name__) + self.assertIn("affinity_floor=mig.migration_affinity_floor(", src, mod.__name__) if __name__ == "__main__": diff --git a/tests/unit/test_multi_attach.py b/tests/unit/test_multi_attach.py new file mode 100644 index 0000000000..b24e4a526c --- /dev/null +++ b/tests/unit/test_multi_attach.py @@ -0,0 +1,647 @@ +"""Multi-attach array LUN mode, phase 2 (simplyblock_core.multi_attach). + + - cluster create/add options: --multi-attach needs --raid-devices, --ftt, + ha_type 'ha'; forces 1+0 and max_fault_tolerance = ftt; --ftt alone is + refused; the regular npcs>=2 rule for FTT2 does not block multi-attach. + - floors: nodes >= FTT+1 (>= 3 for FTT2), >= 1 data device per node. + - device model defaults and the lblk LUN marking. + - host-independent NQN, derived namespace UUID, per-node cntlid ranges. + - RPC wrappers: max_cntlid and stuck_io_timeout_ms sent only when given. + - device stacks (storage_node_ops and device_controller): NQN, cntlid + range, namespace UUID, alceml stuck-I/O timeout; unchanged otherwise. + - peer attach: hold timeouts and multipath mode only for multi-attach. + - owner loopback: cluster map and remote-device connect. +""" + +import json +import unittest +import uuid as uuid_mod +from unittest.mock import MagicMock, patch + +from simplyblock_core import ( + cluster_ops, + constants, + distr_controller, + floors, + multi_attach, + storage_node_ops, +) +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.rpc_client import RPCClient + +DEV_ID = "1b4e28ba-2fa1-41d2-883f-0016d3cca427" + + +def _device(dev_id=DEV_ID, multi=False, node_id="node-a", order=0): + d = NVMeDevice() + d.uuid = dev_id + d.node_id = node_id + d.status = NVMeDevice.STATUS_ONLINE + d.alceml_bdev = f"alceml_{dev_id}" + d.alceml_name = f"alceml_{dev_id}" + d.cluster_device_order = order + d.size = 8 * 1024 ** 3 + d.multi_attach = multi + return d + + +def _ma_cluster(ftt=1): + cl = Cluster() + cl.uuid = "cluster-1" + cl.nqn = f"{constants.CLUSTER_NQN}:cluster-1" + cl.distr_ndcs, cl.distr_npcs = 1, 0 + cl.raid_devices = True + cl.multi_attach = True + cl.ha_type = "ha" + cl.max_fault_tolerance = ftt + cl.status = Cluster.STATUS_ACTIVE + return cl + + +# ---------------------------------------------------------------- options + +class TestResolveOptions(unittest.TestCase): + + def _resolve(self, **kw): + args = dict(multi_attach=True, ftt=1, raid_devices=True, ha_type="ha", + is_single_node=False, distr_ndcs=2, distr_npcs=1, + max_fault_tolerance=1) + args.update(kw) + return cluster_ops._resolve_multi_attach(**args) + + def test_regular_cluster_is_unchanged(self): + self.assertEqual(self._resolve(multi_attach=False, ftt=None), (2, 1, 1)) + + def test_ftt_without_multi_attach_is_refused(self): + with self.assertRaisesRegex(ValueError, "only valid together with multi_attach"): + self._resolve(multi_attach=False, ftt=2) + + def test_forces_one_plus_zero_and_ftt(self): + self.assertEqual(self._resolve(ftt=1), (1, 0, 1)) + self.assertEqual(self._resolve(ftt=2, distr_ndcs=4, distr_npcs=2), (1, 0, 2)) + + def test_requires_raid_devices(self): + with self.assertRaisesRegex(ValueError, "raid_devices"): + self._resolve(raid_devices=False) + + def test_requires_ftt(self): + for ftt in (None, 0, 3): + with self.subTest(ftt=ftt), self.assertRaisesRegex(ValueError, "ftt 1 or 2"): + self._resolve(ftt=ftt) + + def test_requires_ha(self): + with self.assertRaisesRegex(ValueError, "ha_type='ha'"): + self._resolve(ha_type="single") + + def test_refuses_single_node(self): + with self.assertRaisesRegex(ValueError, "single-node"): + self._resolve(is_single_node=True) + + +def _add_cluster(**kw): + """Run _add_cluster_impl with the persistence and side effects stubbed; + returns the Cluster it wrote.""" + written = [] + db = MagicMock() + db.get_clusters.return_value = [MagicMock(cluster_name="other")] + cfg = MagicMock() + cfg.mode = "docker" + cfg.db_connection = "" + cfg.grafana_secret = cluster_ops.SecretStr("") + cfg.grafana_endpoint = "" + cfg.disable_monitoring = True + db.get_deploy_config.return_value = cfg + args = dict(blk_size=4096, page_size_in_blocks=2097152, cap_warn=0, cap_crit=0, + prov_cap_warn=0, prov_cap_crit=0, distr_ndcs=1, distr_npcs=1, + distr_bs=4096, distr_chunk_bs=4096, ha_type="ha", enable_node_affinity=False, + qpair_count=256, max_queue_size=128, inflight_io_threshold=4, + strict_node_anti_affinity=False, is_single_node=False, name="", + max_subsys=10, spdk_vcpu_count=4) + args.update(kw) + with patch.object(cluster_ops, "db_controller", db), \ + patch.object(cluster_ops, "cluster_events"), \ + patch.object(cluster_ops, "validate_spdk_sizing"), \ + patch.object(Cluster, "write_to_db", autospec=True, + side_effect=lambda self, *a, **k: written.append(self)): + cluster_ops._add_cluster_impl(**args) + return written[-1] + + +class TestAddCluster(unittest.TestCase): + + def test_multi_attach_ftt2_is_stored_as_one_plus_zero(self): + cl = _add_cluster(multi_attach=True, ftt=2, raid_devices=True, + distr_ndcs=1, distr_npcs=1, max_fault_tolerance=1) + self.assertTrue(cl.multi_attach) + self.assertTrue(cl.raid_devices) + self.assertEqual((cl.distr_ndcs, cl.distr_npcs), (1, 0)) + self.assertEqual(cl.max_fault_tolerance, 2) + + def test_regular_cluster_keeps_its_scheme(self): + cl = _add_cluster(distr_ndcs=2, distr_npcs=1, max_fault_tolerance=1) + self.assertFalse(cl.multi_attach) + self.assertEqual((cl.distr_ndcs, cl.distr_npcs, cl.max_fault_tolerance), (2, 1, 1)) + + def test_regular_ftt2_still_needs_two_parity_chunks(self): + with self.assertRaisesRegex(ValueError, "distr_npcs >= 2"): + _add_cluster(distr_ndcs=2, distr_npcs=1, max_fault_tolerance=2) + + def test_multi_attach_without_raid_devices_is_refused(self): + with self.assertRaisesRegex(ValueError, "raid_devices"): + _add_cluster(multi_attach=True, ftt=1) + + def test_add_cluster_passes_the_options_through(self): + with patch.object(cluster_ops, "_add_cluster_impl", return_value="c1") as impl: + cluster_ops.add_cluster(4096, 2097152, 0, 0, 0, 0, 1, 1, 4096, 4096, "ha", False, + 256, 128, 4, False, False, "", multi_attach=True, ftt=2, + raid_devices=True) + kw = impl.call_args.kwargs + self.assertEqual((kw["multi_attach"], kw["ftt"]), (True, 2)) + + def test_create_cluster_refuses_ftt_alone(self): + with self.assertRaisesRegex(ValueError, "only valid together"): + cluster_ops.create_cluster( + 4096, 2097152, "", 0, 0, 0, 0, "eth0", "10.0.0.1", 0, 0, "", "", 1, 1, + 4096, 4096, "ha", "docker", False, 256, 256, 128, 4, True, False, "", + "", "", "", "tcp", False, "", ftt=1) + + +class TestCli(unittest.TestCase): + + def _parse(self, *argv): + from simplyblock_cli import cli as cli_module + wrapper = cli_module.CLIWrapper.__new__(cli_module.CLIWrapper) + cli_module.CLIWrapper.__init__(wrapper) + return wrapper.parser.parse_args(list(argv)) + + def test_options_on_create_and_add(self): + for command in ("create", "add"): + with self.subTest(command=command): + args = self._parse("cluster", command, "--raid-devices", "--multi-attach", "--ftt", "2") + self.assertTrue(args.multi_attach) + self.assertEqual(args.ftt, 2) + plain = self._parse("cluster", command) + self.assertFalse(plain.multi_attach) + self.assertIsNone(plain.ftt) + + def test_ftt_choices(self): + with self.assertRaises(SystemExit): + self._parse("cluster", "add", "--ftt", "3") + + +class TestApiV2(unittest.TestCase): + + def test_params_carry_the_options(self): + from simplyblock_web.api.v2.cluster import ClusterParams + p = ClusterParams(max_subsys=10, spdk_vcpu_count=4, raid_devices=True, + multi_attach=True, ftt=2) + dumped = p.model_dump(exclude_none=True) + self.assertEqual((dumped["multi_attach"], dumped["ftt"]), (True, 2)) + self.assertNotIn("ftt", ClusterParams(max_subsys=10, spdk_vcpu_count=4).model_dump(exclude_none=True)) + + +# ---------------------------------------------------------------- floors + +def _node(uid, devices=1, status=StorageNode.STATUS_ONLINE): + n = StorageNode() + n.uuid = uid + n.cluster_id = "cluster-1" + n.status = status + n.nvme_devices = [_device(f"{uid}-d{i}", multi=True, node_id=uid) for i in range(devices)] + return n + + +class TestFloors(unittest.TestCase): + + def test_node_floor_is_ftt_plus_one(self): + f1 = floors.cluster_floors(_ma_cluster(1), False, member_nodes=2) + self.assertEqual((f1.nodes, f1.devices_per_node, f1.data_devices), (2, 1, 1)) + f2 = floors.cluster_floors(_ma_cluster(2), False, member_nodes=2) + self.assertEqual(f2.nodes, 3) + self.assertIn("multi-attach", f2.nodes_rule) + + def test_no_single_or_two_node_override(self): + self.assertEqual(floors.cluster_floors(_ma_cluster(2), False, member_nodes=1).nodes, 3) + + def test_violations_name_the_rule(self): + cl = _ma_cluster(2) + v = floors.evaluate(cl, [_node("a"), _node("b")]) + self.assertTrue(any("FTT+1" in x for x in v), v) + v = floors.evaluate(cl, [_node("a"), _node("b"), _node("c", devices=0)]) + self.assertEqual(len(v), 1) + self.assertIn("one data LUN per node is recommended", v[0]) + self.assertEqual(floors.evaluate(cl, [_node("a"), _node("b", devices=2), _node("c")]), []) + + def test_regular_violation_text_unchanged(self): + cl = _ma_cluster(1) + cl.multi_attach = False + cl.distr_ndcs, cl.distr_npcs = 1, 1 + v = floors.evaluate(cl, [_node("a"), _node("b"), _node("c"), _node("d", devices=0)]) + self.assertIn("node d has 0 online data devices < floor 1 (data devices per node)", v) + + +# ------------------------------------------------------- device model/NQN + +class TestDeviceModel(unittest.TestCase): + + def test_defaults_are_off(self): + d = NVMeDevice() + self.assertEqual((d.multi_attach, d.lun_wwn, d.serving_node_id, d.node_paths), + (False, "", "", {})) + d.node_id = "n1" + self.assertEqual(d.serving_node(), "n1") + d.serving_node_id = "n2" + self.assertEqual(d.serving_node(), "n2") + + def test_record_round_trips(self): + d = _device(multi=True) + d.lun_wwn = "0x600a" + d.node_paths = {"node-a": {"path": "/dev/disk/by-id/wwn-0x600a", "bdev": "aio_x"}} + back = NVMeDevice(json.loads(json.dumps(d.to_dict()))) + self.assertEqual((back.multi_attach, back.lun_wwn, back.node_paths), + (True, "0x600a", d.node_paths)) + + def _devs(self): + a = NVMeDevice({"uuid": "a", "serial_number": "SER-A", "nvme_bdev": "aio_a", + "by_id_path": "/dev/disk/by-id/wwn-0x1", "device_path": "/dev/sdb"}) + b = NVMeDevice({"uuid": "b", "serial_number": "SER-B", "nvme_bdev": "aio_b", + "device_path": "/dev/sdc"}) + j = NVMeDevice({"uuid": "j", "serial_number": "SER-J", "nvme_bdev": "aio_j"}) + entries = [{"serial": "SER-A", "wwn": "0x1"}, {"serial": "SER-B"}, + {"serial": "SER-J", "journal": True}] + return a, b, j, entries + + def test_mark_lun_devices(self): + a, b, j, entries = self._devs() + snode = MagicMock() + snode.get_id.return_value = "node-a" + multi_attach.mark_lun_devices(_ma_cluster(), snode, [a, b, j], entries) + self.assertTrue(a.multi_attach and b.multi_attach) + self.assertEqual((a.lun_wwn, b.lun_wwn), ("0x1", "SER-B")) + self.assertEqual(a.node_paths, {"node-a": {"path": "/dev/disk/by-id/wwn-0x1", "bdev": "aio_a"}}) + self.assertEqual(b.node_paths["node-a"]["path"], "/dev/sdc") + self.assertFalse(j.multi_attach) + + def test_mark_is_a_no_op_on_a_regular_cluster(self): + a, b, j, entries = self._devs() + cl = _ma_cluster() + cl.multi_attach = False + multi_attach.mark_lun_devices(cl, MagicMock(), [a, b], entries) + self.assertFalse(a.multi_attach or b.multi_attach) + self.assertEqual((a.lun_wwn, a.node_paths), ("", {})) + + def test_resolve_lblk_entries_carries_the_wwn(self): + from simplyblock_core import utils + live = {"name": "sdb", "device_path": "/dev/sdb", "serial": "SER-A", "wwn": "0x1", + "by_id_path": "/dev/disk/by-id/wwn-0x1", "size": 1, "type": "disk"} + resolved, missing = utils.resolve_lblk_entries([{"serial": "SER-A", "name": "sdb"}], [live]) + self.assertEqual((resolved[0]["wwn"], missing), ("0x1", [])) + + +class TestNames(unittest.TestCase): + + def test_device_nqn_is_host_independent(self): + cl = _ma_cluster() + snode = MagicMock(subsystem=f"{constants.CLUSTER_NQN}:host1_8080") + nqn = multi_attach.device_subsystem_nqn(snode, _device(multi=True), cl) + self.assertEqual(nqn, f"{constants.CLUSTER_NQN}:lun:{DEV_ID}") + self.assertEqual(multi_attach.device_subsystem_nqn(snode, _device(), cl), + f"{constants.CLUSTER_NQN}:host1_8080:dev:{DEV_ID}") + + def test_namespace_uuid(self): + self.assertEqual(multi_attach.namespace_uuid(_device()), DEV_ID) + ns = multi_attach.namespace_uuid(_device(multi=True)) + self.assertNotEqual(ns, DEV_ID) + self.assertEqual(ns, multi_attach.namespace_uuid(_device(multi=True))) + uuid_mod.UUID(ns) + + def test_cntlid_ranges(self): + self.assertEqual(multi_attach.cntlid_range(0), (1, 64)) + self.assertEqual(multi_attach.cntlid_range(1), (65, 128)) + lo, hi = multi_attach.cntlid_range(constants.MULTI_ATTACH_MAX_NODES - 1) + self.assertLessEqual(hi, 0xFFEF) + for bad in (-1, constants.MULTI_ATTACH_MAX_NODES): + with self.subTest(block=bad), self.assertRaises(ValueError): + multi_attach.cntlid_range(bad) + + def test_kwargs_are_empty_for_regular_devices(self): + d = _device() + self.assertEqual(multi_attach.attach_controller_kwargs(d), {}) + self.assertEqual(multi_attach.alceml_create_kwargs(d), {}) + self.assertEqual(multi_attach.subsystem_create_kwargs(MagicMock(), d), {}) + self.assertEqual(multi_attach.attach_controller_kwargs(MagicMock()), {}) + + def test_hold_and_stuck_io_constants(self): + self.assertLess(constants.MULTI_ATTACH_ALCEML_STUCK_IO_MS, constants.MULTI_ATTACH_HOLD_SEC * 1000) + self.assertEqual(multi_attach.attach_controller_kwargs(_device(multi=True)), { + "ctrlr_loss_timeout_sec": constants.MULTI_ATTACH_HOLD_SEC, "reconnect_delay_sec": 2, + "fast_io_fail_timeout_sec": constants.MULTI_ATTACH_HOLD_SEC}) + self.assertEqual(multi_attach.alceml_create_kwargs(_device(multi=True)), + {"stuck_io_timeout_ms": 60000}) + + +class TestCntlidBlockClaim(unittest.TestCase): + + def _claim(self, alloc, node_id="n3", live=("n1", "n2", "n3")): + from simplyblock_core.db_controller import DBController, _NoTxnStore + kv = {} + if alloc is not None: + kv[b"alloc"] = json.dumps(alloc).encode() + store = MagicMock() + store.get.side_effect = kv.get + store.set.side_effect = kv.__setitem__ + db = DBController.__new__(DBController) + node = StorageNode() + node.uuid = node_id + with patch.object(DBController, "_atomic_update_tx", + side_effect=lambda tr, key, cls, fn, idx: fn(node)): + block = db._claim_cntlid_block_tx(_NoTxnStore(store), b"node", b"alloc", node_id, + set(live), 4, ()) + return block, node, json.loads(kv[b"alloc"]) if b"alloc" in kv else None + + def test_smallest_free_block(self): + block, node, alloc = self._claim({"n1": 0, "n2": 2}) + self.assertEqual((block, node.cntlid_block), (1, 1)) + self.assertEqual(alloc, {"n1": 0, "n2": 2, "n3": 1}) + + def test_first_claim_and_idempotence(self): + self.assertEqual(self._claim(None)[0], 0) + block, _, alloc = self._claim({"n3": 2, "n1": 0}) + self.assertEqual((block, alloc), (2, {"n3": 2, "n1": 0})) + + def test_gone_nodes_release_their_block(self): + block, _, alloc = self._claim({"gone": 0, "n1": 1}, live=("n1", "n3")) + self.assertEqual(block, 0) + self.assertNotIn("gone", alloc) + + def test_full(self): + with self.assertRaisesRegex(ValueError, "at most 4"): + self._claim({"n1": 0, "n2": 1, "x": 2, "y": 3}, live=("n1", "n2", "n3", "x", "y")) + + def test_ensure_uses_an_existing_block(self): + node = StorageNode() + node.cntlid_block = 5 + db = MagicMock() + self.assertEqual(multi_attach.ensure_cntlid_block(node, db), 5) + db.claim_cntlid_block.assert_not_called() + node.cntlid_block = -1 + db.claim_cntlid_block.return_value = 7 + self.assertEqual(multi_attach.ensure_cntlid_block(node, db), 7) + self.assertEqual(node.cntlid_block, 7) + + +# ---------------------------------------------------------------- RPC + +class TestRpcWrappers(unittest.TestCase): + + def _params(self, call): + c = RPCClient("127.0.0.1", 8080, "u", cluster_ops.SecretStr("p")) + with patch.object(c, "_request", return_value=True) as req: + call(c) + return req.call_args.kwargs + + def test_subsystem_create_max_cntlid_only_when_given(self): + p = self._params(lambda c: c.subsystem_create("nqn", "sn", "mn")) + self.assertNotIn("max_cntlid", p) + self.assertEqual(p["min_cntlid"], 1) + p = self._params(lambda c: c.subsystem_create("nqn", "sn", "mn", min_cntlid=65, max_cntlid=128)) + self.assertEqual((p["min_cntlid"], p["max_cntlid"]), (65, 128)) + + def test_alceml_stuck_io_only_when_non_zero(self): + p = self._params(lambda c: c.bdev_alceml_create("a", "b", DEV_ID)) + self.assertNotIn("stuck_io_timeout_ms", p) + p = self._params(lambda c: c.bdev_alceml_create("a", "b", DEV_ID, stuck_io_timeout_ms=60000)) + self.assertEqual(p["stuck_io_timeout_ms"], 60000) + + +# ---------------------------------------------------------- device stacks + +def _stack_snode(): + snode = MagicMock() + snode.subsystem = f"{constants.CLUSTER_NQN}:host1_8080" + snode.enable_test_device = False + snode.cntlid_block = 2 + snode.nvmf_port = 4420 + iface = MagicMock(ip4_address="10.0.0.1", trtype="TCP", listen_address="10.0.0.1") + snode.data_nics = [iface] + return snode + + +class TestStorageDeviceStack(unittest.TestCase): + + def _run(self, dev, cluster): + snode = _stack_snode() + rpc = MagicMock() + rpc.bdev_get.return_value = {"aliases": ["pt-uuid"]} + with patch.object(storage_node_ops, "DBController") as db: + db.return_value.get_cluster_by_id.return_value = cluster + out = storage_node_ops._create_storage_device_stack(rpc, dev, snode, False) + self.assertIs(out, dev) + return snode, rpc + + def test_multi_attach_device(self): + dev = _device(multi=True) + snode, rpc = self._run(dev, _ma_cluster()) + nqn = f"{constants.CLUSTER_NQN}:lun:{DEV_ID}" + rpc.subsystem_create.assert_called_once_with(nqn, "sbcli-cn", DEV_ID, min_cntlid=129, max_cntlid=192) + rpc.nvmf_subsystem_add_ns.assert_called_once_with(nqn, f"alceml_{DEV_ID}_PT", + multi_attach.namespace_uuid(dev), + nguid=multi_attach.namespace_nguid(dev)) + self.assertEqual(snode.create_alceml.call_args.kwargs["stuck_io_timeout_ms"], 60000) + self.assertEqual(dev.nvmf_nqn, nqn) + + def test_regular_device_is_unchanged(self): + dev = _device() + cl = _ma_cluster() + cl.multi_attach = False + snode, rpc = self._run(dev, cl) + nqn = f"{constants.CLUSTER_NQN}:host1_8080:dev:{DEV_ID}" + rpc.subsystem_create.assert_called_once_with(nqn, "sbcli-cn", DEV_ID) + rpc.nvmf_subsystem_add_ns.assert_called_once_with(nqn, f"alceml_{DEV_ID}_PT", DEV_ID) + self.assertNotIn("stuck_io_timeout_ms", snode.create_alceml.call_args.kwargs) + + +class TestDeviceControllerStack(unittest.TestCase): + + def _run(self, dev, cluster): + from simplyblock_core.controllers import device_controller + snode = _stack_snode() + rpc = MagicMock() + rpc.bdev_get.return_value = None + rpc.subsystem_get.return_value = None + snode.rpc_client.return_value = rpc + with patch.object(device_controller, "DBController") as db: + db.return_value.get_cluster_by_id.return_value = cluster + self.assertTrue(device_controller._def_create_device_stack(dev, snode)) + return snode, rpc + + def test_multi_attach_device(self): + dev = _device(multi=True) + snode, rpc = self._run(dev, _ma_cluster()) + nqn = f"{constants.CLUSTER_NQN}:lun:{DEV_ID}" + rpc.subsystem_create.assert_called_once_with(nqn, "sbcli-cn", DEV_ID, min_cntlid=129, max_cntlid=192) + rpc.nvmf_subsystem_add_ns.assert_called_once_with(nqn, f"alceml_{DEV_ID}_PT", + multi_attach.namespace_uuid(dev), + nguid=multi_attach.namespace_nguid(dev)) + self.assertEqual(snode.create_alceml.call_args.kwargs["stuck_io_timeout_ms"], 60000) + self.assertEqual(dev.nvmf_nqn, nqn) + + def test_regular_device_is_unchanged(self): + dev = _device() + cl = _ma_cluster() + cl.multi_attach = False + snode, rpc = self._run(dev, cl) + nqn = f"{constants.CLUSTER_NQN}:host1_8080:dev:{DEV_ID}" + rpc.subsystem_create.assert_called_once_with(nqn, "sbcli-cn", DEV_ID) + rpc.nvmf_subsystem_add_ns.assert_called_once_with(nqn, f"alceml_{DEV_ID}_PT") + self.assertNotIn("stuck_io_timeout_ms", snode.create_alceml.call_args.kwargs) + + +# ---------------------------------------------------------------- attach + +class TestPeerAttach(unittest.TestCase): + + def _attach(self, dev): + dev.nvmf_ip = "10.0.0.5" + dev.nvmf_nqn = "nqn.x" + dev.nvmf_port = 4420 + target = MagicMock(active_rdma=False, active_tcp=True) + rpc = MagicMock() + rpc.bdev_nvme_controller_list.return_value = [] + rpc.bdev_get.return_value = {"name": "b"} + attach = MagicMock() + attach.bdev_nvme_attach_controller.return_value = [f"remote_{dev.alceml_bdev}n1"] + with patch.object(storage_node_ops, "DBController") as db: + db.return_value.get_storage_node_by_id.return_value = target + storage_node_ops._connect_device_attach( + f"remote_{dev.alceml_bdev}", dev, MagicMock(), rpc, attach, ["10.0.0.5"], False) + return attach.bdev_nvme_attach_controller.call_args + + def test_multi_attach_holds_io(self): + call = self._attach(_device(multi=True)) + self.assertEqual(call.kwargs["multipath"], "multipath") + self.assertEqual(call.kwargs["ctrlr_loss_timeout_sec"], constants.MULTI_ATTACH_HOLD_SEC) + self.assertEqual(call.kwargs["reconnect_delay_sec"], constants.MULTI_ATTACH_RECONNECT_DELAY_SEC) + self.assertEqual(call.kwargs["fast_io_fail_timeout_sec"], constants.MULTI_ATTACH_FAST_IO_FAIL_SEC) + + def test_regular_device_is_unchanged(self): + call = self._attach(_device()) + self.assertEqual(call.kwargs, {"multipath": False}) + + +# -------------------------------------------------------------- loopback + +def _map_node(uid, devices): + n = MagicMock(spec=StorageNode) + n.get_id.return_value = uid + n.cluster_id = "cluster-1" + n.status = StorageNode.STATUS_ONLINE + n.is_secondary_node = False + n.failure_domain = -1 + n.nvme_devices = devices + n.remote_devices = [] + return n + + +class TestOwnerLoopback(unittest.TestCase): + + def _map(self, node): + with patch.object(distr_controller, "DBController") as db: + cl = MagicMock(enable_node_affinity=False, enable_failure_domain=False, + sync_replication=False) + db.return_value.get_cluster_by_id.return_value = cl + return distr_controller.get_distr_cluster_map([node], node, "distr-1") + + def _remote(self, dev): + from simplyblock_core.models.nvme_device import RemoteDevice + rd = RemoteDevice() + rd.uuid = dev.uuid + rd.remote_bdev = f"remote_{dev.alceml_bdev}n1" + return rd + + def test_own_multi_attach_device_uses_the_loopback_bdev(self): + dev = _device(multi=True) + node = _map_node("node-a", [dev]) + node.remote_devices = [self._remote(dev)] + entry = self._map(node)["map_cluster"]["node-a"]["devices"][0] + self.assertEqual(entry["bdev_name"], f"remote_alceml_{DEV_ID}n1") + self.assertEqual(entry["status"], NVMeDevice.STATUS_ONLINE) + + def test_own_multi_attach_device_without_loopback_is_unavailable(self): + node = _map_node("node-a", [_device(multi=True)]) + entry = self._map(node)["map_cluster"]["node-a"]["devices"][0] + self.assertEqual(entry["status"], NVMeDevice.STATUS_UNAVAILABLE) + + def test_own_regular_device_uses_the_local_alceml(self): + node = _map_node("node-a", [_device()]) + entry = self._map(node)["map_cluster"]["node-a"]["devices"][0] + self.assertEqual(entry["bdev_name"], f"alceml_{DEV_ID}") + + def test_add_device_map_uses_the_loopback_bdev(self): + dev = _device(multi=True) + target = _map_node("node-a", [dev]) + target.rpc_client.return_value.distr_add_devices.return_value = True + with patch.object(distr_controller, "DBController") as db: + db.return_value.get_storage_node_by_id.return_value = target + with patch.object(distr_controller, "_sync_replication", return_value=False): + distr_controller.send_cluster_map_add_device(dev, target) + cl_map = target.rpc_client.return_value.distr_add_devices.call_args.args[0] + self.assertEqual(cl_map["devices"][0]["bdev_name"], f"remote_alceml_{DEV_ID}n1") + + def _connect(self, own_devices, peer_devices): + this = _map_node("node-a", own_devices) + peer = _map_node("node-b", peer_devices) + rpc = MagicMock() + rpc.bdev_get.side_effect = lambda name: {"name": name} if name.startswith("remote_") else None + this.rpc_client.return_value = rpc + connected = [] + with patch.object(storage_node_ops, "DBController") as db, \ + patch.object(storage_node_ops, "_connect_device_thread", + side_effect=lambda name, dev, node: connected.append(name)), \ + patch.object(storage_node_ops, "_fetch_bdev_name_set", return_value=set()): + db.return_value.get_storage_nodes_by_cluster_id.return_value = [this, peer] + out = storage_node_ops._connect_to_remote_devs(this) + return connected, {rd.get_id() for rd in out} + + def test_connect_includes_own_multi_attach_devices(self): + own = _device("own-1", multi=True, node_id="node-a") + peer = _device("peer-1", multi=True, node_id="node-b", order=1) + connected, ids = self._connect([own], [peer]) + self.assertEqual(sorted(connected), ["remote_alceml_own-1", "remote_alceml_peer-1"]) + self.assertEqual(ids, {"own-1", "peer-1"}) + + def test_connect_skips_own_regular_devices(self): + own = _device("own-1", node_id="node-a") + peer = _device("peer-1", node_id="node-b", order=1) + connected, ids = self._connect([own], [peer]) + self.assertEqual(connected, ["remote_alceml_peer-1"]) + self.assertEqual(ids, {"peer-1"}) + + +if __name__ == "__main__": + unittest.main() + + +class TestNamespaceNguid(unittest.TestCase): + """The NGUID of a multi-attach namespace is set explicitly: SPDK would + otherwise fill it from the PT bdev's name-derived UUID, which collides on + any node that also consumes the LUN.""" + + def test_regular_device_has_none(self): + self.assertIsNone(multi_attach.namespace_nguid(_device())) + + def test_multi_attach_nguid_is_the_namespace_uuid(self): + import uuid as uuid_mod + dev = _device(multi=True) + nguid = multi_attach.namespace_nguid(dev) + self.assertEqual(nguid, uuid_mod.UUID(multi_attach.namespace_uuid(dev)).hex.upper()) + self.assertEqual(len(nguid), 32) + + def test_differs_from_the_pt_bdev_uuid(self): + # bdev_ptnonexcl.c: uuid5(7e25812e-...-000000000001, "_PT"). + import uuid as uuid_mod + dev = _device(multi=True) + pt = uuid_mod.uuid5(uuid_mod.UUID("7e25812e-c8c0-4d3f-8599-000000000001"), f"alceml_{DEV_ID}_PT") + self.assertNotEqual(multi_attach.namespace_nguid(dev), pt.hex.upper()) diff --git a/tests/unit/test_multi_attach_auto_restart.py b/tests/unit/test_multi_attach_auto_restart.py new file mode 100644 index 0000000000..e9e14a9197 --- /dev/null +++ b/tests/unit/test_multi_attach_auto_restart.py @@ -0,0 +1,69 @@ +"""Auto-restart admission on multi-attach clusters: they run 1+0 (npcs 0) and +tolerate ``max_fault_tolerance`` node losses, so two nodes down at FTT 2 must +each be restarted (2026-11-20 FTT2 bed: network cut + SPDK kill left both +offline for good, each blocking the other). Regular clusters keep the npcs +rule.""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core.controllers import tasks_controller as tc +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.storage_node import StorageNode + + +def _node(nid, status): + n = StorageNode() + n.uuid = nid + n.cluster_id = "c1" + n.status = status + n.mgmt_ip = f"10.0.0.{len(nid)}{nid[-1]}" + return n + + +def _cluster(multi, ftt, npcs): + c = Cluster() + c.uuid = "c1" + c.status = Cluster.STATUS_DEGRADED + c.multi_attach = multi + c.max_fault_tolerance = ftt + c.distr_ndcs, c.distr_npcs = (1, npcs) + return c + + +class TestAutoRestartTolerance(unittest.TestCase): + + def _queue(self, cluster, offline_others): + nodes = [_node("n0", StorageNode.STATUS_OFFLINE)] + nodes += [_node(f"n{i + 1}", StorageNode.STATUS_OFFLINE) for i in range(offline_others)] + nodes += [_node(f"m{i}", StorageNode.STATUS_ONLINE) for i in range(3)] + db = MagicMock() + db.get_storage_node_by_id.return_value = nodes[0] + db.get_cluster_by_id.return_value = cluster + db.get_storage_nodes_by_cluster_id.return_value = nodes + with patch.object(tc, "db", db), \ + patch.object(tc, "is_auto_restart_paused", return_value=False), \ + patch.object(tc, "_add_task", return_value="task") as add: + tc.add_node_to_auto_restart(nodes[0]) + return add.called + + def test_multi_attach_ftt2_restarts_with_another_node_down(self): + self.assertTrue(self._queue(_cluster(True, 2, 0), offline_others=1)) + + def test_multi_attach_ftt2_holds_beyond_its_tolerance(self): + # Same rule as npcs on a regular cluster: held once more OTHER nodes + # are down than the cluster tolerates. + self.assertTrue(self._queue(_cluster(True, 2, 0), offline_others=2)) + self.assertFalse(self._queue(_cluster(True, 2, 0), offline_others=3)) + + def test_multi_attach_ftt1(self): + self.assertTrue(self._queue(_cluster(True, 1, 0), offline_others=1)) + self.assertFalse(self._queue(_cluster(True, 1, 0), offline_others=2)) + + def test_regular_cluster_keeps_the_npcs_rule(self): + self.assertTrue(self._queue(_cluster(False, 1, 1), offline_others=1)) + self.assertFalse(self._queue(_cluster(False, 2, 1), offline_others=2)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_multi_attach_cluster_status.py b/tests/unit/test_multi_attach_cluster_status.py new file mode 100644 index 0000000000..2da11818f9 --- /dev/null +++ b/tests/unit/test_multi_attach_cluster_status.py @@ -0,0 +1,52 @@ +"""Cluster status on a multi-attach (1+0 array LUN) cluster: a lost node is +covered by the LUN takeover, so the node-loss tolerance is the cluster's FTT, +not its (zero) parity. Found on the multi-attach bed: the first node loss +suspended the cluster, which shut the other nodes down, takeover standby +included.""" +import unittest +from unittest.mock import MagicMock + +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.services import storage_node_monitor as mon + + +def _cluster(multi_attach=True, ftt=1, ndcs=1, npcs=0): + return MagicMock(multi_attach=multi_attach, max_fault_tolerance=ftt, distr_ndcs=ndcs, + distr_npcs=npcs, enable_failure_domain=False, + strict_node_anti_affinity=False, migration_ft_impact=None) + + +def _verdict(cluster, affected, online_nodes=4, online_devices=4): + return mon._status_verdict_rules(cluster, [], set(range(affected)), online_nodes, + online_devices, []) + + +class TestMultiAttachNodeTolerance(unittest.TestCase): + + def test_healthy_is_active(self): + self.assertEqual(_verdict(_cluster(), 0), Cluster.STATUS_ACTIVE) + + def test_one_lost_node_with_ftt1_is_degraded(self): + self.assertEqual(_verdict(_cluster(), 1, online_nodes=3), Cluster.STATUS_DEGRADED) + + def test_two_lost_nodes_with_ftt1_suspend(self): + # each LUN is attached to two nodes only + self.assertEqual(_verdict(_cluster(), 2, online_nodes=2), Cluster.STATUS_SUSPENDED) + + def test_two_lost_nodes_with_ftt2_is_degraded(self): + self.assertEqual(_verdict(_cluster(ftt=2), 2, online_nodes=2), Cluster.STATUS_DEGRADED) + + def test_regular_no_parity_cluster_unchanged(self): + # 1+0 without multi-attach: no takeover, a lost node suspends as before + self.assertEqual(_verdict(_cluster(multi_attach=False), 1, online_nodes=3), + Cluster.STATUS_SUSPENDED) + + def test_regular_parity_cluster_unchanged(self): + self.assertEqual(_verdict(_cluster(multi_attach=False, ndcs=2, npcs=1), 1, online_nodes=3), + Cluster.STATUS_DEGRADED) + self.assertEqual(_verdict(_cluster(multi_attach=False, ndcs=2, npcs=1), 2, online_nodes=2), + Cluster.STATUS_SUSPENDED) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_multi_attach_device_events.py b/tests/unit/test_multi_attach_device_events.py new file mode 100644 index 0000000000..5e88392e34 --- /dev/null +++ b/tests/unit/test_multi_attach_device_events.py @@ -0,0 +1,92 @@ +"""Distrib device events on multi-attach clusters: the home node of a LUN +served by another node reaches it remotely, like any peer. Its I/O error is +about that path and must not fail the LUN for the whole cluster (2026-11-20 +FTT2 bed: a home node's error_read on the dead serving node's export, +processed after the takeover, marked the taken-over LUN unavailable and +blocked every LVS of the 1+0 cluster).""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import main_distr_event_collector as collector + + +def _run(serving_node_id, multi=True, event_node="node-1"): + dev = NVMeDevice() + dev.uuid = "dev-1" + dev.node_id = "node-1" + dev.cluster_device_order = 3 + dev.status = NVMeDevice.STATUS_ONLINE + dev.multi_attach = multi + dev.serving_node_id = serving_node_id + + home = MagicMock() + home.get_id.return_value = "node-1" + home.status = StorageNode.STATUS_ONLINE + home.cluster_id = "c1" + home.nvme_devices = [dev] + + peer = MagicMock() + peer.get_id.return_value = "node-3" + peer.status = StorageNode.STATUS_ONLINE + peer.cluster_id = "c1" + peer.nvme_devices = [] + + event = MagicMock() + event.message = "error_read" + event.node_id = event_node + event.storage_id = 3 + event.object_dict = {} + + db = MagicMock() + db.get_storage_node_by_id.side_effect = lambda nid: peer if nid == "node-3" else home + db.get_storage_nodes_by_cluster_id.return_value = [home, peer] + with patch.object(collector, "db", db), \ + patch.object(collector.device_controller, "device_set_unavailable") as unavailable, \ + patch.object(collector.device_controller, "device_set_io_error"), \ + patch.object(collector.distr_controller, "send_dev_status_event") as peer_event, \ + patch.object(collector, "remove_remote_device_from_node") as removed, \ + patch.object(NVMeDevice, "lock_device_connection"), \ + patch.object(NVMeDevice, "release_device_connection"), \ + patch.object(NVMeDevice, "is_connection_in_progress_to_node", return_value=False): + collector.process_device_event(event, collector.logger) + _run.removed = removed + _run.status = event.status + return unavailable, peer_event + + +class TestHomeNodeOfLunServedElsewhere(unittest.TestCase): + + def test_home_node_error_is_left_to_the_takeover(self): + unavailable, peer_event = _run(serving_node_id="node-2") + unavailable.assert_not_called() + peer_event.assert_not_called() + _run.removed.assert_not_called() + self.assertEqual(_run.status, "skipped:multi_attach_remote_path") + + def test_peer_error_is_left_to_the_takeover(self): + # 2026-11-21 FTT2 bed: per-node "unavailable" sent during a takeover + # stayed in the node's map; on 1+0 its reads failed and SPDK aborted. + unavailable, peer_event = _run(serving_node_id="node-2", event_node="node-3") + unavailable.assert_not_called() + peer_event.assert_not_called() + _run.removed.assert_not_called() + + def test_regular_cluster_peer_error_is_processed_as_before(self): + _run(serving_node_id="", multi=False, event_node="node-3") + self.assertNotEqual(_run.status, "skipped:multi_attach_remote_path") + + def test_home_node_serving_its_lun_fails_it(self): + unavailable, peer_event = _run(serving_node_id="") + unavailable.assert_called_once() + peer_event.assert_not_called() + + def test_regular_cluster_unchanged(self): + unavailable, _ = _run(serving_node_id="node-2", multi=False) + unavailable.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_multi_attach_port_check.py b/tests/unit/test_multi_attach_port_check.py new file mode 100644 index 0000000000..390c0296df --- /dev/null +++ b/tests/unit/test_multi_attach_port_check.py @@ -0,0 +1,110 @@ +"""Port checks on multi-attach clusters (1+0): a port SPDK reports blocked is +its own fence, and one bad device fences every LVS of the cluster at once +(2026-11-20 FTT2 bed: a stale error event marked a taken-over LUN +unavailable; three healthy nodes were set DOWN within 4 s, the node floor +broke and the cluster suspended). + + A. the node's own port gets the fence grace peer-owned ports have; + B. more nodes than the cluster tolerates failing only the port check at + once are not set DOWN. +Regular clusters keep the 1 s re-read and set DOWN as before.""" + +import time +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.services import storage_node_monitor as snm +from tests.unit.test_port_check_deliberate_fence import OWN_PORT, _node + + +def _cluster(multi, ftt=2): + c = Cluster() + c.uuid = "c1" + c.status = Cluster.STATUS_ACTIVE + c.multi_attach = multi + c.max_fault_tolerance = ftt + return c + + +class TestOwnPortGrace(unittest.TestCase): + + def setUp(self): + snm._blocked_port_since.clear() + self.addCleanup(snm._blocked_port_since.clear) + + def _check(self, cluster, reads): + node = _node("n1") + db = MagicMock() + db.get_primary_storage_nodes_by_secondary_node_id.return_value = [] + db.get_cluster_by_id.return_value = cluster + reads = list(reads) + + def _ports(_n, ports): + own = reads.pop(0) if reads else True + return {p: (own if p == OWN_PORT else True) for p in ports} + sleep = MagicMock() + with patch.object(snm, "db", db), \ + patch.object(snm.health_controller, "check_ports_on_node", side_effect=_ports), \ + patch.object(snm, "_remediate_stale_port_blocks"), \ + patch.object(snm.time, "sleep", sleep): + return snm.node_port_check_fun(node), sleep + + def test_fresh_own_fence_is_not_counted(self): + verdict, sleep = self._check(_cluster(True), [False]) + self.assertTrue(verdict) + sleep.assert_not_called() + + def test_own_fence_past_the_grace_counts(self): + snm._blocked_port_since[("n1", OWN_PORT)] = time.monotonic() - snm.PEER_PORT_FENCE_GRACE_SEC - 1 + verdict, _ = self._check(_cluster(True), [False]) + self.assertFalse(verdict) + + def test_regular_cluster_keeps_the_re_read(self): + verdict, sleep = self._check(_cluster(False), [False, False]) + self.assertFalse(verdict) + sleep.assert_called_once_with(snm.PORT_BLOCK_CONFIRM_SEC) + + +class TestCorrelatedPortFailure(unittest.TestCase): + + def setUp(self): + snm._port_failed_at.clear() + snm._correlated_event_at.clear() + self.addCleanup(snm._port_failed_at.clear) + + def _fail(self, *nids): + for nid in nids: + snm._note_port_check(_node(nid), False) + + def test_within_tolerance_the_node_goes_down(self): + self._fail("n1", "n2") + self.assertEqual(snm._correlated_port_failure(_cluster(True, ftt=2), _node("n2")), "") + + def test_beyond_tolerance_no_node_goes_down(self): + self._fail("n1", "n2", "n3") + with patch.object(snm.storage_events, "snode_port_check_correlated") as ev: + reason = snm._correlated_port_failure(_cluster(True, ftt=2), _node("n3")) + snm._correlated_port_failure(_cluster(True, ftt=2), _node("n1")) + self.assertIn("3 nodes fail only their port check", reason) + ev.assert_called_once() # rate-limited per cluster + + def test_a_passing_check_clears_the_node(self): + self._fail("n1", "n2", "n3") + snm._note_port_check(_node("n1"), True) + self.assertEqual(snm._correlated_port_failure(_cluster(True, ftt=2), _node("n2")), "") + + def test_old_failures_age_out(self): + self._fail("n1", "n2", "n3") + for nid in ("n1", "n2"): + cid, _t = snm._port_failed_at[nid] + snm._port_failed_at[nid] = (cid, time.monotonic() - snm.PORT_FAILURE_CORRELATION_SEC - 1) + self.assertEqual(snm._correlated_port_failure(_cluster(True, ftt=2), _node("n3")), "") + + def test_regular_cluster_never_holds(self): + self._fail("n1", "n2", "n3", "n4") + self.assertEqual(snm._correlated_port_failure(_cluster(False, ftt=1), _node("n1")), "") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_node_full_write_lun_fields.py b/tests/unit/test_node_full_write_lun_fields.py new file mode 100644 index 0000000000..c329c52613 --- /dev/null +++ b/tests/unit/test_node_full_write_lun_fields.py @@ -0,0 +1,108 @@ +"""A full-object write of a StorageNode keeps the stored multi-attach LUN +ownership fields of its devices (serving_node_id, fenced_node_ids, +array_attached_node_ids): only atomic_update changes them. + +2026-11-20 FTT2 bed (net cut sn0 + SPDK kill sn1): the reintegration +unfenced a node at 21:32:37.754; the owner's restart, holding a copy loaded +before that, wrote its full record five times in the next 1.4 s and restored +fenced_node_ids. The node stayed fenced and the case never healed.""" + +import json +import unittest + +from simplyblock_core.models.base_model import BaseModel +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode + + +def _node(fenced, serving="", attached=(), status=StorageNode.STATUS_ONLINE, multi=True): + n = StorageNode() + n.uuid = "owner" + n.status = status + d = NVMeDevice() + d.uuid = "dev-1" + d.node_id = "owner" + d.multi_attach = multi + d.fenced_node_ids = list(fenced) + d.serving_node_id = serving + d.array_attached_node_ids = list(attached) + n.nvme_devices = [d] + return n + + +class _Tr: + def __init__(self, stored): + self.data = {b"k": json.dumps(stored.to_dict(unwrap_secrets=True)).encode()} if stored else {} + + def get(self, key): + raw = self.data.get(key) + + class _V: + def wait(self_inner): + return self_inner + + def present(self_inner): + return raw is not None + + def __bytes__(self_inner): + return raw + return _V() + + def set(self, key, value): + self.data[key] = value + + +def _write(stored, stale): + tr = _Tr(stored) + BaseModel._write_tx(tr, b"k", json.dumps(stale.to_dict(unwrap_secrets=True)).encode(), + StorageNode, stale, [], None, None) + return StorageNode().from_dict(json.loads(tr.data[b"k"])).nvme_devices[0] + + +class TestFullWriteKeepsLunOwnership(unittest.TestCase): + + def test_stale_copy_does_not_refence(self): + stored = _node(fenced=[], serving="node-b", attached=["node-a", "node-b"]) + stale = _node(fenced=["node-a"], serving="node-b", attached=["node-b"], + status=StorageNode.STATUS_RESTARTING) + dev = _write(stored, stale) + self.assertEqual(dev.fenced_node_ids, []) + self.assertEqual(dev.array_attached_node_ids, ["node-a", "node-b"]) + + def test_stale_copy_does_not_undo_a_takeover(self): + dev = _write(_node(fenced=["owner"], serving="node-b"), _node(fenced=[], serving="")) + self.assertEqual(dev.serving_node_id, "node-b") + self.assertEqual(dev.fenced_node_ids, ["owner"]) + + def test_other_fields_are_written(self): + stored = _node(fenced=[]) + stale = _node(fenced=[], status=StorageNode.STATUS_RESTARTING) + stale.nvme_devices[0].nvmf_nqn = "nqn.new" + tr = _Tr(stored) + BaseModel._write_tx(tr, b"k", json.dumps(stale.to_dict(unwrap_secrets=True)).encode(), + StorageNode, stale, [], None, None) + n = StorageNode().from_dict(json.loads(tr.data[b"k"])) + self.assertEqual(n.status, StorageNode.STATUS_RESTARTING) + self.assertEqual(n.nvme_devices[0].nvmf_nqn, "nqn.new") + + def test_new_record_is_written_as_is(self): + dev = _write(None, _node(fenced=["x"], serving="y")) + self.assertEqual((dev.fenced_node_ids, dev.serving_node_id), (["x"], "y")) + + def test_regular_node_is_untouched(self): + # No multi-attach device: no stored-record read, the write as before. + stored, stale = _node(fenced=[], multi=False), _node(fenced=["x"], multi=False) + self.assertFalse(stale.needs_full_write_merge()) + tr = _Tr(stored) + reads = [] + get = tr.get + tr.get = lambda key: (reads.append(key), get(key))[1] + BaseModel._write_tx(tr, b"k", json.dumps(stale.to_dict(unwrap_secrets=True)).encode(), + StorageNode, stale, [], None, None) + self.assertEqual(reads, []) + dev = StorageNode().from_dict(json.loads(tr.data[b"k"])).nvme_devices[0] + self.assertEqual(dev.fenced_node_ids, ["x"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index e3283bb557..2e38c5c41a 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -19,6 +19,8 @@ import unittest from unittest.mock import DEFAULT, MagicMock, call, patch +import pytest + from simplyblock_core import storage_node_ops from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.nvme_device import JMDevice, NVMeDevice, RemoteJMDevice @@ -26,6 +28,16 @@ from simplyblock_core.rpc_client import RPCConnectionError, RPCException, RPCRemoteError from tests._mocks import unique_ip + +@pytest.fixture(autouse=True) +def _floors_out_of_the_removal_tests(): + """These fixtures (one to four nodes, no devices) model the removal + mechanics, not the resource floors (simplyblock_core.floors), which would + refuse every removal here; the floors have their own tests + (test_ftt_protection.TestFloorGate, test_cluster_floors).""" + with patch.object(storage_node_ops.floors, "evaluate", return_value=[]): + yield + # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- diff --git a/tests/unit/test_removal_three_steps.py b/tests/unit/test_removal_three_steps.py index b2e2f50343..284a126427 100644 --- a/tests/unit/test_removal_three_steps.py +++ b/tests/unit/test_removal_three_steps.py @@ -83,7 +83,9 @@ def test_k1_is_degraded_only_by_the_removal_while_its_devices_rebuild(self): def test_k2_stays_active_while_the_removal_rebuilds(self): leaving = _node("x", "10.0.9.9", StorageNode.STATUS_MIGRATING_DEVICES, (NVMeDevice.STATUS_FAILED,)) - status, _ = self._verdicts(_cluster(npcs=2), self._online(5) + [leaving]) + # Six online nodes: the 2+2 node floor (n+2k), so the floors do not + # degrade the verdict on their own. + status, _ = self._verdicts(_cluster(npcs=2), self._online(6) + [leaving]) assert status == Cluster.STATUS_ACTIVE def test_k1_the_removals_own_shutdown_is_already_the_removals(self): diff --git a/tests/unit/test_san_host_agent.py b/tests/unit/test_san_host_agent.py new file mode 100644 index 0000000000..864a4db50f --- /dev/null +++ b/tests/unit/test_san_host_agent.py @@ -0,0 +1,172 @@ +"""Node agent, SAN host side: the initiator report (how an array knows the +host) and the scan for newly presented LUNs.""" +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from simplyblock_web import node_utils + + +class _Root(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.root) + real_read, real_isdir, real_listdir = node_utils._read_sysfs, os.path.isdir, os.listdir + + def _map(p): + return os.path.join(self.root, p.lstrip("/")) if p.startswith("/") else p + for target, fn in ((node_utils, "_read_sysfs"),): + p = patch.object(target, fn, side_effect=lambda path: real_read(_map(path))) + p.start() + self.addCleanup(p.stop) + p = patch.object(node_utils.os.path, "isdir", side_effect=lambda path: real_isdir(_map(path))) + p.start() + self.addCleanup(p.stop) + p = patch.object(node_utils.os, "listdir", side_effect=lambda path: real_listdir(_map(path))) + p.start() + self.addCleanup(p.stop) + self.map = _map + + def mk(self, rel, content=""): + path = os.path.join(self.root, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as fh: + fh.write(content) + + +class TestInitiators(_Root): + + def test_reports_iqn_wwpns_and_hostnqn(self): + self.mk("etc/iscsi/initiatorname.iscsi", + "## generated\n#InitiatorName=iqn.ignored\nInitiatorName=iqn.1994-05.com.redhat:abc\n") + self.mk("sys/class/fc_host/host3/port_name", "0x21000024FF4C1A2B\n") + self.mk("sys/class/fc_host/host4/port_name", "0x0000000000000000\n") # unconfigured port + self.mk("etc/nvme/hostnqn", "nqn.2014-08.org.nvmexpress:uuid:1234\n") + with patch.object(node_utils.os.path, "exists", return_value=False): + out = node_utils.get_san_initiators() + self.assertEqual(out, { + "iscsi_iqn": "iqn.1994-05.com.redhat:abc", + "fc_wwpns": ["21000024ff4c1a2b"], + "nvme_hostnqn": "nqn.2014-08.org.nvmexpress:uuid:1234", + }) + + def test_kubernetes_agent_reads_the_hosts_etc(self): + self.mk("host/etc/nvme/hostnqn", "nqn.host") + self.mk("etc/nvme/hostnqn", "nqn.container") + with patch.object(node_utils.os.path, "exists", return_value=False): + self.assertEqual(node_utils.get_san_initiators()["nvme_hostnqn"], "nqn.host") + + def test_powerflex_sdc_guid(self): + with patch.object(node_utils.os.path, "exists", + side_effect=lambda p: p == "/opt/emc/scaleio/sdc/bin/drv_cfg"), \ + patch.object(node_utils.shell_utils, "run_command", + return_value=("INFO: DRV_CFG: Query GUID: 0E9B4C8A-1234-4ABC-9DEF-00112233AABB", + "", 0)): + self.assertEqual(node_utils.get_san_initiators()["sdc_guid"], + "0E9B4C8A-1234-4ABC-9DEF-00112233AABB") + + def test_nothing_configured(self): + with patch.object(node_utils.os.path, "exists", return_value=False): + self.assertEqual(node_utils.get_san_initiators(), {}) + + +class TestScan(_Root): + + def test_scans_scsi_hosts_and_nvme_controllers(self): + written = {} + paths = ["/sys/class/scsi_host/host0/scan", "/sys/class/scsi_host/host7/scan", + "/sys/class/nvme/nvme2/rescan_controller"] + with patch("glob.glob", side_effect=lambda pat: [p for p in paths if p.startswith(pat.split("*")[0])]), \ + patch.object(node_utils, "_write_sysfs", side_effect=lambda p, v: written.__setitem__(p, v)): + out = node_utils.scan_for_new_luns() + self.assertEqual(out["scanned"], paths) + self.assertEqual(written["/sys/class/scsi_host/host7/scan"], "- - -") + self.assertEqual(written["/sys/class/nvme/nvme2/rescan_controller"], "1") + + def test_a_failing_host_does_not_stop_the_scan(self): + paths = ["/sys/class/scsi_host/host0/scan", "/sys/class/scsi_host/host1/scan"] + + def write(p, v): + if p.endswith("host0/scan"): + raise OSError("busy") + with patch("glob.glob", side_effect=lambda pat: [p for p in paths if "scsi_host" in pat]), \ + patch.object(node_utils, "_write_sysfs", side_effect=write): + self.assertEqual(node_utils.scan_for_new_luns()["scanned"], paths[1:]) + + +# A SCSI LUN reached over several paths (FC or iSCSI with dm-multipath) is +# one multipath map: the inventory reports the map, with the identity of its +# paths, and a LUN lookup returns the map instead of the paths below it. +_MPATH_LSBLK = {"blockdevices": [ + {"name": "sda", "kname": "sda", "type": "disk", "size": 100 << 30, "serial": "LOCAL1", "wwn": None}, + {"name": "sdb", "kname": "sdb", "type": "disk", "size": 4 << 40, "serial": "624a9370aa", "wwn": "0x624a9370aa", + "children": [{"name": "mpatha", "kname": "dm-3", "type": "mpath", "size": 4 << 40}]}, + {"name": "sdc", "kname": "sdc", "type": "disk", "size": 4 << 40, "serial": "624a9370aa", "wwn": "0x624a9370aa", + "children": [{"name": "mpatha", "kname": "dm-3", "type": "mpath", "size": 4 << 40}]}, + {"name": "sdd", "kname": "sdd", "type": "disk", "size": 2 << 40, "serial": "624a9370bb", "wwn": "0x624a9370bb"}, +]} + + +class TestMultipathInventory(unittest.TestCase): + + def setUp(self): + import json as _json + patches = [ + patch.object(node_utils.shell_utils, "run_command", + return_value=(_json.dumps(_MPATH_LSBLK), "", 0)), + patch.object(node_utils, "_root_disk_names", return_value={"sda"}), + patch.object(node_utils.host_network, "node_hostname", return_value="h1"), + patch.object(node_utils, "read_persistent_links", return_value={ + "dm-3": ["/dev/disk/by-id/dm-uuid-mpath-3624a9370aa", "/dev/disk/by-id/dm-name-mpatha"], + "sdd": ["/dev/disk/by-id/wwn-0x624a9370bb"]}), + patch.object(node_utils, "_read_sysfs", return_value=""), + patch.object(node_utils, "_logical_block_size", return_value=512), + patch.object(node_utils, "_physical_block_size", return_value=4096), + patch.object(node_utils, "_disk_holders", + side_effect=lambda n: ["dm-3"] if n in ("sdb", "sdc") else []), + patch.object(node_utils, "_subtree_mounted", return_value=False), + patch.object(node_utils, "simplyblock_data_signature", return_value=False), + ] + for p in patches: + p.start() + self.addCleanup(p.stop) + + def test_the_map_is_reported_with_its_paths_identity(self): + devices = {d["name"]: d for d in node_utils.get_block_devices_info()} + m = devices["mpatha"] + self.assertEqual((m["type"], m["kname"], m["device_path"]), ("mpath", "dm-3", "/dev/mapper/mpatha")) + self.assertEqual((m["serial"], m["wwn"], m["size"]), ("624a9370aa", "0x624a9370aa", 4 << 40)) + self.assertEqual(m["paths"], ["sdb", "sdc"]) + self.assertEqual(m["by_id_path"], "/dev/disk/by-id/dm-uuid-mpath-3624a9370aa") + self.assertEqual(m["holders"], []) + # The paths stay listed, held by the map, so nothing takes them alone. + self.assertEqual(devices["sdb"]["multipath_map"], "mpatha") + self.assertEqual(devices["sdb"]["holders"], ["dm-3"]) + + def test_a_lun_lookup_returns_the_map_not_its_paths(self): + found = node_utils.find_lun_devices("naa.624a9370aa") + self.assertEqual([d["name"] for d in found], ["mpatha"]) + self.assertEqual(found[0]["device_path"], "/dev/mapper/mpatha") + self.assertEqual(found[0]["by_id_path"], "/dev/disk/by-id/dm-uuid-mpath-3624a9370aa") + + def test_a_single_path_lun_is_still_its_disk(self): + found = node_utils.find_lun_devices("624a9370bb") + self.assertEqual([d["name"] for d in found], ["sdd"]) + + +class TestMultipathResolution(unittest.TestCase): + + def test_a_configured_lun_resolves_to_its_map_not_a_path(self): + from simplyblock_core import utils + path = {"type": "disk", "serial": "624a9370aa", "size": 1, "holders": ["dm-3"], + "multipath_map": "mpatha"} + live = [dict(path, name="sdb", device_path="/dev/sdb"), + {"type": "mpath", "name": "mpatha", "kname": "dm-3", "serial": "624a9370aa", "size": 1, + "device_path": "/dev/mapper/mpatha", "holders": []}, + dict(path, name="sdc", device_path="/dev/sdc")] + resolved, missing = utils.resolve_lblk_entries([{"name": "mpatha", "serial": "624a9370aa"}], live) + self.assertEqual(missing, []) + self.assertEqual(resolved[0]["current_path"], "/dev/mapper/mpatha") + self.assertEqual(resolved[0]["io_engine"], "uring") diff --git a/tests/unit/test_spdk_crash_logs.py b/tests/unit/test_spdk_crash_logs.py new file mode 100644 index 0000000000..54652f8e9f --- /dev/null +++ b/tests/unit/test_spdk_crash_logs.py @@ -0,0 +1,58 @@ +"""The node agent keeps the log tail of an SPDK container that died on its own +before a restart removes it (2026-11-20 FTT2 bed: three SPDKs died in a +cascade and every trace went with their containers).""" + +import json +import os +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_web.api.internal.storage_node import docker as agent + + +def _client(status, logs=b"line1\nline2\n"): + c = MagicMock() + c.attrs = {"State": {"Status": status, "ExitCode": 134, "OOMKilled": False, + "Error": "", "StartedAt": "s", "FinishedAt": "f"}} + c.logs.return_value = logs + client = MagicMock() + client.containers.get.return_value = c + return client + + +class TestPreserveExitedSpdkLogs(unittest.TestCase): + + def setUp(self): + self.dir = tempfile.mkdtemp() + p = patch.object(agent, "SPDK_CRASH_LOG_DIR", self.dir) + p.start() + self.addCleanup(p.stop) + + def test_exited_container_log_is_kept(self): + path = agent._preserve_exited_spdk_logs(_client("exited"), "/spdk_4420") + self.assertTrue(path and os.path.exists(path)) + with open(path, "rb") as f: + head, rest = f.read().split(b"\n", 1) + self.assertEqual(json.loads(head)["ExitCode"], 134) + self.assertIn(b"line2", rest) + + def test_running_container_is_left_alone(self): + self.assertIsNone(agent._preserve_exited_spdk_logs(_client("running"), "/spdk_4420")) + self.assertEqual(os.listdir(self.dir), []) + + def test_only_the_newest_are_kept(self): + with patch.object(agent, "SPDK_CRASH_LOG_KEEP", 2): + for i in range(4): + with patch.object(agent.time, "strftime", return_value=f"t{i}"): + agent._preserve_exited_spdk_logs(_client("exited"), "/spdk_4420") + self.assertEqual(len(os.listdir(self.dir)), 2) + + def test_never_raises(self): + client = MagicMock() + client.containers.get.side_effect = RuntimeError("dockerd wedged") + self.assertIsNone(agent._preserve_exited_spdk_logs(client, "/spdk_4420")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_sync_replication_site_loss.py b/tests/unit/test_sync_replication_site_loss.py index 3d76a422db..614b6431d4 100644 --- a/tests/unit/test_sync_replication_site_loss.py +++ b/tests/unit/test_sync_replication_site_loss.py @@ -17,6 +17,15 @@ from simplyblock_core.services import tasks_runner_sync_promote as runner from simplyblock_core.snode_client import SNodeClientException + +@pytest.fixture(autouse=True) +def _floors_out_of_the_site_tests(): + """The site verdicts are judged on the veto / majority rules alone; the + resource floors (simplyblock_core.floors, DEGRADED while a site's online + nodes are below n+2k) are tested in test_cluster_floors.""" + with patch.object(storage_node_monitor.floors, "evaluate", return_value=[]): + yield + API_UP = "the node API reports the SPDK process up" RPC_ANSWERS = "its RPC answers" diff --git a/tests/unit/test_two_node_c1_fixes.py b/tests/unit/test_two_node_c1_fixes.py index b135e03a36..fc4fd8f5fd 100644 --- a/tests/unit/test_two_node_c1_fixes.py +++ b/tests/unit/test_two_node_c1_fixes.py @@ -361,3 +361,15 @@ def test_connect_entry_tells_the_csi_driver_it_is_two_node(arbitrated): cluster=_Cluster(arbitrated), host_entry=None, host_nqn=None) assert entry.two_node is arbitrated assert entry.model_dump(by_alias=True)["two-node"] is arbitrated + + +def test_keep_alive_for_multi_attach_clusters(): + """1+0 multi-attach: a hard-reset node's own volumes waited 10-14 s for the + 8 s keep-alive (FTT1/FTT2 beds, 2026-11-20).""" + cl = _Cluster(False) + cl.multi_attach = True + entry = build_nvme_connect_entry(transport="tcp", ip="10.0.0.1", port=4420, nqn="nqn.x", + ctrl_loss_tmo=constants.LVOL_NVME_CONNECT_CTRL_LOSS_TMO, + cluster=cl, host_entry=None, host_nqn=None) + assert entry.keep_alive_tmo == constants.MULTI_ATTACH_CLIENT_KEEP_ALIVE_TO == 3 + assert "--keep-alive-tmo=3" in entry.connect diff --git a/tests/unit/web/api/v2/test_device_endpoints.py b/tests/unit/web/api/v2/test_device_endpoints.py index b42cdef191..8e9f131b9b 100644 --- a/tests/unit/web/api/v2/test_device_endpoints.py +++ b/tests/unit/web/api/v2/test_device_endpoints.py @@ -374,82 +374,79 @@ def test_detail_dispatches_watch_device(self, client, device, device_controller, CLUSTER_ID, STORAGE_NODE_ID, DEVICE_ID) -RESIZE_RESULT = { - 'device_id': DEVICE_ID, 'old_size': 100, 'new_size': 200, 'placement_size': 100, - 'resized': True, 'uniform': False, 'warning': 'grown unevenly', +GROWN_DEVICE = { + 'device_id': DEVICE_ID, 'old_size': 100, 'new_size': 150, 'backend_size': 200, + 'unused_size': 50, 'placement_size': 100, } -class TestResizeDevice: - - def test_returns_resize_result(self, client, db, device, device_controller): - device_controller.resize_device.return_value = RESIZE_RESULT - - response = client.post(f'{BASE}/{DEVICE_ID}/resize') - - assert response.status_code == 200 - assert response.json() == RESIZE_RESULT - device_controller.resize_device.assert_called_once_with(DEVICE_ID) - - def test_refusal_returns_409(self, client, db, device, device_controller): - device_controller.resize_device.side_effect = ValueError('shrank') +class TestResizeDeviceRemoved: + def test_single_device_resize_is_gone(self, client, db, device, device_controller): response = client.post(f'{BASE}/{DEVICE_ID}/resize') - assert response.status_code == 409 - assert 'shrank' in response.json()['detail'] + assert response.status_code in (404, 405) class TestGrowClusterDevices: URL = f'/api/v2/clusters/{CLUSTER_ID}/grow-devices' - def test_grows_devices_of_lblk_cluster(self, client, db, cluster, monkeypatch): + def _mock(self, monkeypatch, result): from unittest.mock import MagicMock import simplyblock_web.api.v2.cluster as cluster_module mock = MagicMock() - mock.grow_cluster_devices.return_value = {'devices': [RESIZE_RESULT], 'errors': {}, 'uniform': True} + mock.grow_cluster_devices.return_value = result monkeypatch.setattr(cluster_module, 'device_controller', mock) + return mock + + def test_grows_devices_of_lblk_cluster(self, client, db, cluster, monkeypatch): + mock = self._mock(monkeypatch, { + 'grown': True, 'factor': 1.5, 'devices': [GROWN_DEVICE], 'errors': {}, 'reason': ''}) cluster.device_mode = 'lblk' response = client.post(self.URL) assert response.status_code == 200 - assert response.json()['uniform'] is True - assert response.json()['rebalance'] is None - mock.grow_cluster_devices.assert_called_once_with(CLUSTER_ID, rebalance=True) - - def test_uneven_growth_reports_rebalance(self, client, db, cluster, monkeypatch): - from unittest.mock import MagicMock + body = response.json() + assert body['grown'] is True + assert body['factor'] == 1.5 + assert body['devices'] == [GROWN_DEVICE] + mock.grow_cluster_devices.assert_called_once_with(CLUSTER_ID) + + def test_lun_factor_is_passed_through(self, client, db, cluster, monkeypatch): + mock = self._mock(monkeypatch, { + 'grown': True, 'factor': 2.0, 'devices': [], 'errors': {}, 'reason': '', + 'luns': {'d1': {'lun_id': 'vol-1', 'old_size': 1, 'new_size': 2}}}) + cluster.device_mode = 'lblk' - import simplyblock_web.api.v2.cluster as cluster_module - mock = MagicMock() - mock.grow_cluster_devices.return_value = { - 'devices': [RESIZE_RESULT], 'errors': {}, 'uniform': False, - 'rebalance': {'weights_updated': True, 'started': True, 'reason': ''}} - monkeypatch.setattr(cluster_module, 'device_controller', mock) + response = client.post(f'{self.URL}?lun_factor=2') + assert response.status_code == 200 + assert response.json()['luns']['d1']['lun_id'] == 'vol-1' + mock.grow_cluster_devices.assert_called_once_with(CLUSTER_ID, lun_factor=2.0) + assert client.post(f'{self.URL}?lun_factor=1').status_code == 422 + + def test_nothing_grown_reports_reason_and_errors(self, client, db, cluster, monkeypatch): + self._mock(monkeypatch, { + 'grown': False, 'factor': 1.0, 'devices': [], 'errors': {DEVICE_ID: 'shrank'}, + 'reason': 'A block device is smaller than its device; nothing was grown'}) cluster.device_mode = 'lblk' response = client.post(self.URL) assert response.status_code == 200 - assert response.json()['rebalance'] == {'weights_updated': True, 'started': True, 'reason': ''} + assert response.json()['grown'] is False + assert response.json()['errors'] == {DEVICE_ID: 'shrank'} - def test_rebalance_can_be_disabled(self, client, db, cluster, monkeypatch): - from unittest.mock import MagicMock - - import simplyblock_web.api.v2.cluster as cluster_module - mock = MagicMock() - mock.grow_cluster_devices.return_value = { - 'devices': [], 'errors': {}, 'uniform': False, 'rebalance': None} - monkeypatch.setattr(cluster_module, 'device_controller', mock) + def test_rebalance_parameter_is_gone(self, client, db, cluster, monkeypatch): + mock = self._mock(monkeypatch, { + 'grown': False, 'factor': 1.0, 'devices': [], 'errors': {}, 'reason': 'x'}) cluster.device_mode = 'lblk' - response = client.post(self.URL, params={'rebalance': 'false'}) + client.post(self.URL, params={'rebalance': 'false'}) - assert response.status_code == 200 - mock.grow_cluster_devices.assert_called_once_with(CLUSTER_ID, rebalance=False) + mock.grow_cluster_devices.assert_called_once_with(CLUSTER_ID) def test_nvme_cluster_returns_409(self, client, db, cluster): cluster.device_mode = 'nvme'