From 5dba85915382e78033c54b686cfca725f4fc8b0c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 28 May 2026 20:01:52 +0000 Subject: [PATCH 01/10] feat(block-storage): add EBS suite contract + AWS reference scripts Adds a provider-agnostic block-storage validation suite plus the AWS reference implementation covering the block-storage trio that share one fixture (launch instance + create/attach EBS volume): - DATASVC-XX-02 (#321) snapshot_lifecycle: CreateSnapshot -> restore to a new volume -> attach/mount -> byte-compare the sentinel data. - DATASVC-XX-03 (#322) volume_resize: ModifyVolume + in-guest growpart/resize2fs, verifying the guest sees the larger filesystem. - DATASVC-XX-04 (#323) volume_persistence: stop/start the instance and verify the volume stays attached with its data intact. Reuses existing StepSuccessCheck + FieldExistsCheck + CrudOperationsCheck (no new validation classes, no released_tests.json change). common/ebs.py centralizes the boto3 volume/snapshot lifecycle and the Nitro NVMe by-id device mapping; create_volume/teardown_volume own the shared fixture. Signed-off-by: Cursor Agent Co-authored-by: Alexandre Begnoche --- .../providers/aws/config/block-storage.yaml | 172 ++++++++++ .../scripts/block-storage/create_volume.py | 193 +++++++++++ .../block-storage/snapshot_lifecycle.py | 181 ++++++++++ .../scripts/block-storage/teardown_volume.py | 82 +++++ .../block-storage/volume_persistence.py | 145 ++++++++ .../scripts/block-storage/volume_resize.py | 179 ++++++++++ .../providers/aws/scripts/common/ebs.py | 322 ++++++++++++++++++ isvctl/configs/suites/block-storage.yaml | 119 +++++++ 8 files changed, 1393 insertions(+) create mode 100644 isvctl/configs/providers/aws/config/block-storage.yaml create mode 100644 isvctl/configs/providers/aws/scripts/block-storage/create_volume.py create mode 100644 isvctl/configs/providers/aws/scripts/block-storage/snapshot_lifecycle.py create mode 100644 isvctl/configs/providers/aws/scripts/block-storage/teardown_volume.py create mode 100644 isvctl/configs/providers/aws/scripts/block-storage/volume_persistence.py create mode 100644 isvctl/configs/providers/aws/scripts/block-storage/volume_resize.py create mode 100644 isvctl/configs/providers/aws/scripts/common/ebs.py create mode 100644 isvctl/configs/suites/block-storage.yaml diff --git a/isvctl/configs/providers/aws/config/block-storage.yaml b/isvctl/configs/providers/aws/config/block-storage.yaml new file mode 100644 index 00000000..ba8552a4 --- /dev/null +++ b/isvctl/configs/providers/aws/config/block-storage.yaml @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# AWS Block Storage (EBS) Validation Configuration +# +# Imports validations from the provider-agnostic block-storage canonical +# config. This file only provides AWS-specific commands (boto3 + SSH) and +# settings. The launch_instance / teardown steps reuse the VM reference +# scripts; the volume steps live in scripts/block-storage/. +# +# Steps: +# SETUP +# 1. launch_instance - boto3: provision an instance for the fixture +# 2. create_volume - boto3 + SSH: create/attach/format/mount/seed an EBS volume +# TEST +# 3. snapshot_lifecycle - DATASVC-XX-02 (#321): snapshot -> restore -> verify data +# 4. volume_resize - DATASVC-XX-03 (#322): ModifyVolume + growpart/resize2fs +# 5. volume_persistence - DATASVC-XX-04 (#323): stop/start -> verify reattach + data +# TEARDOWN +# 6. teardown_volume - boto3: detach + delete the fixture volume +# 7. teardown - boto3: terminate the instance + SG + key pair +# +# Usage: +# uv run isvctl test run -f isvctl/configs/providers/aws/config/block-storage.yaml +# +# Required IAM Permissions: +# ec2:DescribeInstances, ec2:RunInstances, ec2:TerminateInstances, +# ec2:StopInstances, ec2:StartInstances, ec2:DescribeImages, ec2:DescribeSubnets, +# ec2:DescribeVpcs, ec2:CreateKeyPair, ec2:DeleteKeyPair, ec2:CreateSecurityGroup, +# ec2:DeleteSecurityGroup, ec2:AuthorizeSecurityGroupIngress, ec2:CreateTags, +# ec2:CreateVolume, ec2:DeleteVolume, ec2:DescribeVolumes, ec2:AttachVolume, +# ec2:DetachVolume, ec2:ModifyVolume, ec2:DescribeVolumesModifications, +# ec2:CreateSnapshot, ec2:DeleteSnapshot, ec2:DescribeSnapshots + +import: + - ../../../suites/block-storage.yaml + +version: "1.0" + +commands: + block_storage: + phases: ["setup", "test", "teardown"] + steps: + # AWS-specific: launch the fixture instance (reuses the VM script) + - name: launch_instance + phase: setup + command: "python3 ../scripts/vm/launch_instance.py" + args: + - "--name" + - "isv-test-block" + - "--instance-type" + - "{{instance_type}}" + - "--region" + - "{{region}}" + timeout: 600 + + # AWS-specific: create + attach + format + mount + seed an EBS volume + - name: create_volume + phase: setup + command: "python3 ../scripts/block-storage/create_volume.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + - "--size-gib" + - "{{volume_size_gib}}" + timeout: 600 + + # DATASVC-XX-02: snapshot -> restore -> verify data + - name: snapshot_lifecycle + phase: test + command: "python3 ../scripts/block-storage/snapshot_lifecycle.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--volume-id" + - "{{steps.create_volume.volume_id}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + - "--expected-content" + - "{{steps.create_volume.sentinel_content}}" + timeout: 900 + + # DATASVC-XX-03: ModifyVolume + in-guest growpart/resize2fs + - name: volume_resize + phase: test + command: "python3 ../scripts/block-storage/volume_resize.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--volume-id" + - "{{steps.create_volume.volume_id}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + - "--mount-point" + - "{{steps.create_volume.mount_point}}" + timeout: 900 + + # DATASVC-XX-04: stop/start -> verify reattach + data + # Runs last so the stop/start IP churn does not affect the other tests. + - name: volume_persistence + phase: test + command: "python3 ../scripts/block-storage/volume_persistence.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--volume-id" + - "{{steps.create_volume.volume_id}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + - "--mount-point" + - "{{steps.create_volume.mount_point}}" + - "--expected-content" + - "{{steps.create_volume.sentinel_content}}" + timeout: 900 + + # AWS-specific: detach + delete the fixture volume + - name: teardown_volume + phase: teardown + command: "python3 ../scripts/block-storage/teardown_volume.py" + args: + - "--region" + - "{{region}}" + - "--volume-id" + - "{{steps.create_volume.volume_id}}" + - "{{teardown_flag}}" + timeout: 300 + + # AWS-specific: terminate the instance (reuses the VM script) + - name: teardown + phase: teardown + command: "python3 ../scripts/vm/teardown.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--delete-key-pair" + - "--delete-security-group" + - "{{teardown_flag}}" + timeout: 600 + +tests: + cluster_name: "aws-block-storage-validation" + description: "AWS block storage (EBS) validation tests" + + settings: + region: "us-west-2" + instance_type: "m6i.large" + volume_size_gib: "10" + teardown_flag: "{{(env.BLOCK_STORAGE_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" diff --git a/isvctl/configs/providers/aws/scripts/block-storage/create_volume.py b/isvctl/configs/providers/aws/scripts/block-storage/create_volume.py new file mode 100644 index 00000000..6aaccb34 --- /dev/null +++ b/isvctl/configs/providers/aws/scripts/block-storage/create_volume.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Block-storage fixture: create, attach, format, mount, and seed a volume. + +Shared setup step for the block-storage suite (DATASVC-XX-02/03/04). It +creates an EBS volume in the instance's AZ, attaches it, partitions + +formats it (ext4), mounts it, and writes a sentinel file. The volume ID, +mount point, and sentinel content are passed to the snapshot / resize / +persistence test steps, which all reuse this single fixture. + +Output JSON: +{ + "success": true, + "platform": "block_storage", + "test_name": "create_volume", + "volume_id": "vol-xxx", + "device": "/dev/sdf", + "mount_point": "/mnt/isv-block", + "size_gib": 10, + "sentinel_path": "/mnt/isv-block/isv-sentinel.txt", + "sentinel_content": "isv-ncp-validate-block-storage-...", + "operations": { + "create": {"passed": true}, + "attach": {"passed": true}, + "format": {"passed": true}, + "mount": {"passed": true}, + "write_sentinel": {"passed": true} + } +} +""" + +import argparse +import json +import os +import sys +import uuid +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # providers/aws/scripts/ (for common.*) + +import boto3 +from botocore.exceptions import ClientError, NoCredentialsError +from common import ebs +from common.ssh_utils import ssh_run, wait_for_ssh + +# Remote setup: resolve the attached device via its stable by-id symlink, +# lay down a single GPT partition, format it ext4, mount it, and write the +# sentinel file. Placeholders are substituted in Python (avoids f-string vs +# shell brace conflicts). +_SETUP_SCRIPT = r""" +set -euo pipefail +BYID="__BYID__" +MOUNT="__MOUNT__" +SENTINEL="$MOUNT/isv-sentinel.txt" +CONTENT="__CONTENT__" +for _ in $(seq 1 30); do [ -e "$BYID" ] && break; sleep 2; done +DEV=$(readlink -f "$BYID") +sudo parted -s "$DEV" mklabel gpt +sudo parted -s "$DEV" mkpart primary ext4 0% 100% +sudo partprobe "$DEV" || true +sudo udevadm settle || true +for _ in $(seq 1 30); do [ -e "__BYID__-part1" ] && break; sleep 2; done +PART=$(readlink -f "__BYID__-part1") +sudo mkfs.ext4 -F "$PART" +sudo mkdir -p "$MOUNT" +sudo mount "$PART" "$MOUNT" +echo -n "$CONTENT" | sudo tee "$SENTINEL" >/dev/null +sudo sync +""" + + +def _render(script: str, **subs: str) -> str: + """Substitute __NAME__ placeholders in a remote shell script template.""" + for key, value in subs.items(): + script = script.replace(f"__{key}__", value) + return script + + +def _fail(op: dict[str, Any], message: str) -> None: + """Mark an operation failed with a message.""" + op["passed"] = False + op["error"] = message + + +def main() -> int: + """Create, attach, format, mount, and seed a block volume; print JSON.""" + parser = argparse.ArgumentParser(description="Block-storage fixture: create + attach + seed a volume") + parser.add_argument("--instance-id", required=True, help="EC2 instance to attach the volume to") + parser.add_argument("--region", default=os.environ.get("AWS_REGION", "us-west-2")) + parser.add_argument("--key-file", required=True, help="Path to SSH private key") + parser.add_argument("--ssh-user", default="ubuntu", help="SSH username") + parser.add_argument("--size-gib", type=int, default=10, help="Volume size in GiB") + parser.add_argument("--device", default="/dev/sdf", help="Requested attach device name") + parser.add_argument("--mount-point", default="/mnt/isv-block", help="In-guest mount point") + args = parser.parse_args() + + sentinel_content = f"isv-ncp-validate-block-storage-{uuid.uuid4().hex}" + operations: dict[str, dict[str, Any]] = { + "create": {"passed": False}, + "attach": {"passed": False}, + "format": {"passed": False}, + "mount": {"passed": False}, + "write_sentinel": {"passed": False}, + } + result: dict[str, Any] = { + "success": False, + "platform": "block_storage", + "test_name": "create_volume", + "instance_id": args.instance_id, + "volume_id": None, + "device": args.device, + "mount_point": args.mount_point, + "size_gib": args.size_gib, + "sentinel_path": f"{args.mount_point}/isv-sentinel.txt", + "sentinel_content": sentinel_content, + "operations": operations, + } + + ec2 = boto3.client("ec2", region_name=args.region) + + try: + instances = ec2.describe_instances(InstanceIds=[args.instance_id]) + instance = instances["Reservations"][0]["Instances"][0] + availability_zone = instance["Placement"]["AvailabilityZone"] + public_ip = instance.get("PublicIpAddress") + result["availability_zone"] = availability_zone + + volume_id = ebs.create_volume(ec2, availability_zone, args.size_gib, name="isv-validate-block") + result["volume_id"] = volume_id + ebs.wait_for_volume_available(ec2, volume_id) + operations["create"]["passed"] = True + + ebs.attach_volume(ec2, volume_id, args.instance_id, args.device) + ebs.wait_for_volume_in_use(ec2, volume_id) + operations["attach"]["passed"] = True + except (ClientError, NoCredentialsError) as e: + result["error"] = f"Volume create/attach failed: {e}" + print(json.dumps(result, indent=2)) + return 1 + + if not public_ip: + result["error"] = "Instance has no public IP for SSH" + print(json.dumps(result, indent=2)) + return 1 + + if not wait_for_ssh(public_ip, args.ssh_user, args.key_file, max_attempts=30, interval=10): + result["error"] = "SSH not ready on fixture instance" + print(json.dumps(result, indent=2)) + return 1 + + if not ebs.wait_for_attachment_device(public_ip, args.ssh_user, args.key_file, volume_id): + _fail(operations["format"], "Attached volume did not appear in guest") + result["error"] = "Attached volume device never appeared in guest" + print(json.dumps(result, indent=2)) + return 1 + + setup = _render( + _SETUP_SCRIPT, + BYID=ebs.guest_by_id_path(volume_id), + MOUNT=args.mount_point, + CONTENT=sentinel_content, + ) + rc, _, err = ssh_run(public_ip, args.ssh_user, args.key_file, setup, timeout=180) + if rc == 0: + operations["format"]["passed"] = True + operations["mount"]["passed"] = True + operations["write_sentinel"]["passed"] = True + else: + _fail(operations["format"], f"Guest format/mount/seed failed (rc={rc}): {err.strip()[:300]}") + result["error"] = "Guest format/mount/seed failed" + + result["success"] = all(op["passed"] for op in operations.values()) + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/aws/scripts/block-storage/snapshot_lifecycle.py b/isvctl/configs/providers/aws/scripts/block-storage/snapshot_lifecycle.py new file mode 100644 index 00000000..2450495f --- /dev/null +++ b/isvctl/configs/providers/aws/scripts/block-storage/snapshot_lifecycle.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DATASVC-XX-02: verify volume snapshots (issue #321). + +Snapshots the fixture volume, restores the snapshot to a brand-new volume, +attaches + mounts that restore on the same instance, and byte-compares the +sentinel file against the content the fixture wrote. A successful round-trip +proves the snapshot captured the data and the restore is independently +usable. The restored volume and snapshot are cleaned up in a finally block. + +Output JSON: +{ + "success": true, + "platform": "block_storage", + "test_name": "snapshot_lifecycle", + "volume_id": "vol-source", + "snapshot_id": "snap-xxx", + "restored_volume_id": "vol-restore", + "operations": { + "create_snapshot": {"passed": true}, + "restore_volume": {"passed": true}, + "verify_data": {"passed": true, "content_matches": true} + } +} +""" + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # providers/aws/scripts/ (for common.*) + +import boto3 +from botocore.exceptions import ClientError, NoCredentialsError +from common import ebs +from common.ssh_utils import ssh_run, wait_for_ssh + + +def _fail(op: dict[str, Any], message: str) -> None: + """Mark an operation failed with a message.""" + op["passed"] = False + op["error"] = message + + +def main() -> int: + """Snapshot the fixture volume, restore it, verify the data; print JSON.""" + parser = argparse.ArgumentParser(description="Verify volume snapshots (DATASVC-XX-02)") + parser.add_argument("--instance-id", required=True, help="EC2 instance for the restored volume") + parser.add_argument("--region", default=os.environ.get("AWS_REGION", "us-west-2")) + parser.add_argument("--volume-id", required=True, help="Source (fixture) volume to snapshot") + parser.add_argument("--key-file", required=True, help="Path to SSH private key") + parser.add_argument("--ssh-user", default="ubuntu", help="SSH username") + parser.add_argument("--expected-content", required=True, help="Sentinel content written by the fixture") + parser.add_argument("--restore-device", default="/dev/sdg", help="Attach device for the restored volume") + parser.add_argument("--restore-mount", default="/mnt/isv-restored", help="Mount point for the restored volume") + args = parser.parse_args() + + operations: dict[str, dict[str, Any]] = { + "create_snapshot": {"passed": False}, + "restore_volume": {"passed": False}, + "verify_data": {"passed": False}, + } + result: dict[str, Any] = { + "success": False, + "platform": "block_storage", + "test_name": "snapshot_lifecycle", + "volume_id": args.volume_id, + "snapshot_id": None, + "restored_volume_id": None, + "operations": operations, + } + + ec2 = boto3.client("ec2", region_name=args.region) + + snapshot_id: str | None = None + restored_volume_id: str | None = None + try: + instances = ec2.describe_instances(InstanceIds=[args.instance_id]) + instance = instances["Reservations"][0]["Instances"][0] + availability_zone = instance["Placement"]["AvailabilityZone"] + public_ip = instance.get("PublicIpAddress") + + # Flush the guest page cache so the snapshot is consistent with the + # sentinel write (best-effort; the fixture already synced after write). + if public_ip: + ssh_run(public_ip, args.ssh_user, args.key_file, "sudo sync") + + try: + snapshot_id = ebs.create_snapshot(ec2, args.volume_id, name="isv-validate-snap") + result["snapshot_id"] = snapshot_id + ebs.wait_for_snapshot_completed(ec2, snapshot_id) + operations["create_snapshot"]["passed"] = True + except (ClientError, NoCredentialsError) as e: + _fail(operations["create_snapshot"], str(e)) + result["error"] = f"CreateSnapshot failed: {e}" + return _emit(result) + + try: + restored_volume_id = ebs.create_volume_from_snapshot(ec2, snapshot_id, availability_zone) + result["restored_volume_id"] = restored_volume_id + ebs.wait_for_volume_available(ec2, restored_volume_id) + ebs.attach_volume(ec2, restored_volume_id, args.instance_id, args.restore_device) + ebs.wait_for_volume_in_use(ec2, restored_volume_id) + operations["restore_volume"]["passed"] = True + except ClientError as e: + _fail(operations["restore_volume"], str(e)) + result["error"] = f"Restore failed: {e}" + return _emit(result, ec2, restored_volume_id, snapshot_id) + + if not public_ip or not wait_for_ssh(public_ip, args.ssh_user, args.key_file, max_attempts=30, interval=10): + _fail(operations["verify_data"], "SSH not ready for restore verification") + result["error"] = "SSH not ready" + return _emit(result, ec2, restored_volume_id, snapshot_id) + + if not ebs.wait_for_attachment_device(public_ip, args.ssh_user, args.key_file, restored_volume_id): + _fail(operations["verify_data"], "Restored volume did not appear in guest") + return _emit(result, ec2, restored_volume_id, snapshot_id) + + rc, out, err = ebs.mount_and_read_sentinel( + public_ip, args.ssh_user, args.key_file, restored_volume_id, args.restore_mount + ) + if rc != 0: + _fail(operations["verify_data"], f"Could not read sentinel on restore (rc={rc}): {err.strip()[:300]}") + return _emit(result, ec2, restored_volume_id, snapshot_id) + + content_matches = out.strip() == args.expected_content.strip() + operations["verify_data"]["content_matches"] = content_matches + if content_matches: + operations["verify_data"]["passed"] = True + else: + _fail(operations["verify_data"], "Restored sentinel does not match fixture content") + + result["success"] = all(op["passed"] for op in operations.values()) + return _emit(result, ec2, restored_volume_id, snapshot_id) + except (ClientError, NoCredentialsError) as e: + result["error"] = str(e) + return _emit(result, ec2, restored_volume_id, snapshot_id) + + +def _emit( + result: dict[str, Any], + ec2: Any = None, + restored_volume_id: str | None = None, + snapshot_id: str | None = None, +) -> int: + """Best-effort cleanup of the restored volume + snapshot, then print JSON.""" + if ec2 is not None and restored_volume_id: + err = ebs.detach_and_delete_volume(ec2, restored_volume_id) + if err: + result.setdefault("cleanup_errors", []).append(err) + if ec2 is not None and snapshot_id: + err = ebs.delete_snapshot_best_effort(ec2, snapshot_id) + if err: + result.setdefault("cleanup_errors", []).append(err) + if result.get("cleanup_errors"): + result["success"] = False + cleanup_msg = f"Cleanup failed: {'; '.join(result['cleanup_errors'])}" + result["error"] = f"{result['error']}; {cleanup_msg}" if result.get("error") else cleanup_msg + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/aws/scripts/block-storage/teardown_volume.py b/isvctl/configs/providers/aws/scripts/block-storage/teardown_volume.py new file mode 100644 index 00000000..ba9d1162 --- /dev/null +++ b/isvctl/configs/providers/aws/scripts/block-storage/teardown_volume.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Detach and delete the block-storage fixture volume (teardown). + +Best-effort cleanup of the volume created by ``create_volume.py``. Runs +before the instance teardown so the volume is removed explicitly rather +than left dangling. The restored volume and snapshot from the snapshot +test clean themselves up in their own step. + +Output JSON: +{ + "success": true, + "platform": "block_storage", + "test_name": "teardown_volume", + "resources_deleted": ["vol-xxx"] +} +""" + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # providers/aws/scripts/ (for common.*) + +import boto3 +from common import ebs + + +def main() -> int: + """Detach and delete the fixture volume; print JSON result.""" + parser = argparse.ArgumentParser(description="Teardown block-storage fixture volume") + parser.add_argument("--region", default=os.environ.get("AWS_REGION", "us-west-2")) + parser.add_argument("--volume-id", required=True, help="Fixture volume to delete") + parser.add_argument("--skip-destroy", action="store_true", help="Skip actual destroy") + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "block_storage", + "test_name": "teardown_volume", + "resources_deleted": [], + } + + if args.skip_destroy: + result["success"] = True + result["message"] = f"Volume {args.volume_id} preserved (--skip-destroy); delete manually when done" + print(json.dumps(result, indent=2)) + return 0 + + ec2 = boto3.client("ec2", region_name=args.region) + error = ebs.detach_and_delete_volume(ec2, args.volume_id) + if error: + result["error"] = error + result["message"] = f"Failed to delete volume {args.volume_id}" + else: + result["success"] = True + result["resources_deleted"].append(args.volume_id) + result["message"] = f"Volume {args.volume_id} deleted" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/aws/scripts/block-storage/volume_persistence.py b/isvctl/configs/providers/aws/scripts/block-storage/volume_persistence.py new file mode 100644 index 00000000..5bf0812d --- /dev/null +++ b/isvctl/configs/providers/aws/scripts/block-storage/volume_persistence.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DATASVC-XX-04: persistent volumes survive instance restarts (issue #323). + +Stops and starts the instance, then confirms the attached block volume is +still attached after the restart and that its sentinel data is intact. The +volume is re-mounted in the guest (a manual mount does not survive a reboot) +and the sentinel file is byte-compared against the fixture content. + +Output JSON: +{ + "success": true, + "platform": "block_storage", + "test_name": "volume_persistence", + "volume_id": "vol-xxx", + "operations": { + "stop": {"passed": true}, + "start": {"passed": true}, + "verify_attached": {"passed": true}, + "verify_data": {"passed": true, "content_matches": true} + } +} +""" + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # providers/aws/scripts/ (for common.*) + +import boto3 +from botocore.exceptions import ClientError, NoCredentialsError +from common import ebs +from common.ec2 import wait_for_public_ip +from common.ssh_utils import wait_for_ssh + + +def _fail(op: dict[str, Any], message: str) -> None: + """Mark an operation failed with a message.""" + op["passed"] = False + op["error"] = message + + +def main() -> int: + """Stop/start the instance and verify the volume + data survive; print JSON.""" + parser = argparse.ArgumentParser(description="Verify block volume persistence across restart (DATASVC-XX-04)") + parser.add_argument("--instance-id", required=True, help="EC2 instance to restart") + parser.add_argument("--region", default=os.environ.get("AWS_REGION", "us-west-2")) + parser.add_argument("--volume-id", required=True, help="Volume expected to persist (fixture volume)") + parser.add_argument("--key-file", required=True, help="Path to SSH private key") + parser.add_argument("--ssh-user", default="ubuntu", help="SSH username") + parser.add_argument("--mount-point", default="/mnt/isv-block", help="In-guest mount point") + parser.add_argument("--expected-content", required=True, help="Sentinel content written by the fixture") + args = parser.parse_args() + + operations: dict[str, dict[str, Any]] = { + "stop": {"passed": False}, + "start": {"passed": False}, + "verify_attached": {"passed": False}, + "verify_data": {"passed": False}, + } + result: dict[str, Any] = { + "success": False, + "platform": "block_storage", + "test_name": "volume_persistence", + "volume_id": args.volume_id, + "instance_id": args.instance_id, + "operations": operations, + } + + ec2 = boto3.client("ec2", region_name=args.region) + + try: + ec2.stop_instances(InstanceIds=[args.instance_id]) + ec2.get_waiter("instance_stopped").wait( + InstanceIds=[args.instance_id], WaiterConfig={"Delay": 15, "MaxAttempts": 40} + ) + operations["stop"]["passed"] = True + + ec2.start_instances(InstanceIds=[args.instance_id]) + ec2.get_waiter("instance_status_ok").wait( + InstanceIds=[args.instance_id], WaiterConfig={"Delay": 15, "MaxAttempts": 40} + ) + instances = ec2.describe_instances(InstanceIds=[args.instance_id]) + instance = instances["Reservations"][0]["Instances"][0] + public_ip = instance.get("PublicIpAddress") or wait_for_public_ip(ec2, args.instance_id) + if not public_ip: + _fail(operations["start"], "No public IP after restart") + result["error"] = "No public IP after restart" + print(json.dumps(result, indent=2)) + return 1 + if not wait_for_ssh(public_ip, args.ssh_user, args.key_file, max_attempts=40, interval=15): + _fail(operations["start"], "SSH not ready after restart") + result["error"] = "SSH not ready after restart" + print(json.dumps(result, indent=2)) + return 1 + operations["start"]["passed"] = True + except (ClientError, NoCredentialsError) as e: + result["error"] = f"Restart failed: {e}" + print(json.dumps(result, indent=2)) + return 1 + + if ebs.is_volume_attached_to(ec2, args.volume_id, args.instance_id): + operations["verify_attached"]["passed"] = True + else: + _fail(operations["verify_attached"], "Volume not attached after restart") + + if operations["verify_attached"]["passed"]: + rc, out, err = ebs.mount_and_read_sentinel( + public_ip, args.ssh_user, args.key_file, args.volume_id, args.mount_point + ) + if rc != 0: + _fail(operations["verify_data"], f"Could not read sentinel after restart (rc={rc}): {err.strip()[:300]}") + else: + content_matches = out.strip() == args.expected_content.strip() + operations["verify_data"]["content_matches"] = content_matches + if content_matches: + operations["verify_data"]["passed"] = True + else: + _fail(operations["verify_data"], "Sentinel content changed across restart") + + result["success"] = all(op["passed"] for op in operations.values()) + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/aws/scripts/block-storage/volume_resize.py b/isvctl/configs/providers/aws/scripts/block-storage/volume_resize.py new file mode 100644 index 00000000..25ba166e --- /dev/null +++ b/isvctl/configs/providers/aws/scripts/block-storage/volume_resize.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DATASVC-XX-03: verify volume resizing (issue #322). + +Grows the fixture volume with ModifyVolume, then grows the in-guest +partition (growpart) and filesystem (resize2fs) and confirms the larger +capacity is visible to the guest. Online resize keeps the volume mounted +throughout, so no data is lost. + +Output JSON: +{ + "success": true, + "platform": "block_storage", + "test_name": "volume_resize", + "volume_id": "vol-xxx", + "old_size_gib": 10, + "new_size_gib": 15, + "fs_bytes_before": 10434441216, + "fs_bytes_after": 15728623616, + "operations": { + "modify_volume": {"passed": true}, + "grow_partition": {"passed": true}, + "resize_filesystem": {"passed": true}, + "verify_size": {"passed": true} + } +} +""" + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # providers/aws/scripts/ (for common.*) + +import boto3 +from botocore.exceptions import ClientError, NoCredentialsError +from common import ebs +from common.ssh_utils import ssh_run, wait_for_ssh + +_READ_FS_BYTES = r""" +set -euo pipefail +MOUNT="__MOUNT__" +df -B1 --output=size "$MOUNT" | tail -1 | tr -d ' ' +""" + +_GROWPART = r""" +set -euo pipefail +DEV=$(readlink -f "__BYID__") +sudo growpart "$DEV" 1 +sudo partprobe "$DEV" || true +sudo udevadm settle || true +""" + +_RESIZE2FS = r""" +set -euo pipefail +PART=$(readlink -f "__BYID__-part1") +sudo resize2fs "$PART" +""" + + +def _fail(op: dict[str, Any], message: str) -> None: + """Mark an operation failed with a message.""" + op["passed"] = False + op["error"] = message + + +def main() -> int: + """Grow the fixture volume + in-guest filesystem and verify; print JSON.""" + parser = argparse.ArgumentParser(description="Verify volume resizing (DATASVC-XX-03)") + parser.add_argument("--instance-id", required=True, help="EC2 instance the volume is attached to") + parser.add_argument("--region", default=os.environ.get("AWS_REGION", "us-west-2")) + parser.add_argument("--volume-id", required=True, help="Volume to resize (fixture volume)") + parser.add_argument("--key-file", required=True, help="Path to SSH private key") + parser.add_argument("--ssh-user", default="ubuntu", help="SSH username") + parser.add_argument("--mount-point", default="/mnt/isv-block", help="In-guest mount point") + parser.add_argument("--grow-gib", type=int, default=5, help="GiB to add to the volume") + args = parser.parse_args() + + operations: dict[str, dict[str, Any]] = { + "modify_volume": {"passed": False}, + "grow_partition": {"passed": False}, + "resize_filesystem": {"passed": False}, + "verify_size": {"passed": False}, + } + result: dict[str, Any] = { + "success": False, + "platform": "block_storage", + "test_name": "volume_resize", + "volume_id": args.volume_id, + "operations": operations, + } + + ec2 = boto3.client("ec2", region_name=args.region) + by_id = ebs.guest_by_id_path(args.volume_id) + + try: + instances = ec2.describe_instances(InstanceIds=[args.instance_id]) + public_ip = instances["Reservations"][0]["Instances"][0].get("PublicIpAddress") + + volumes = ec2.describe_volumes(VolumeIds=[args.volume_id]) + old_size = volumes["Volumes"][0]["Size"] + new_size = old_size + args.grow_gib + result["old_size_gib"] = old_size + result["new_size_gib"] = new_size + + try: + ebs.modify_volume_size(ec2, args.volume_id, new_size) + ebs.wait_for_modification_complete(ec2, args.volume_id) + operations["modify_volume"]["passed"] = True + except (ClientError, NoCredentialsError, RuntimeError) as e: + _fail(operations["modify_volume"], str(e)) + result["error"] = f"ModifyVolume failed: {e}" + print(json.dumps(result, indent=2)) + return 1 + except (ClientError, NoCredentialsError) as e: + result["error"] = str(e) + print(json.dumps(result, indent=2)) + return 1 + + if not public_ip or not wait_for_ssh(public_ip, args.ssh_user, args.key_file, max_attempts=30, interval=10): + result["error"] = "SSH not ready for in-guest resize" + print(json.dumps(result, indent=2)) + return 1 + + rc, before_out, _ = ssh_run(public_ip, args.ssh_user, args.key_file, _READ_FS_BYTES.replace("__MOUNT__", args.mount_point)) + fs_before = int(before_out.strip()) if rc == 0 and before_out.strip().isdigit() else None + result["fs_bytes_before"] = fs_before + + rc, _, err = ssh_run(public_ip, args.ssh_user, args.key_file, _GROWPART.replace("__BYID__", by_id)) + if rc == 0: + operations["grow_partition"]["passed"] = True + else: + _fail(operations["grow_partition"], f"growpart failed (rc={rc}): {err.strip()[:300]}") + + if operations["grow_partition"]["passed"]: + rc, _, err = ssh_run(public_ip, args.ssh_user, args.key_file, _RESIZE2FS.replace("__BYID__", by_id)) + if rc == 0: + operations["resize_filesystem"]["passed"] = True + else: + _fail(operations["resize_filesystem"], f"resize2fs failed (rc={rc}): {err.strip()[:300]}") + + rc, after_out, _ = ssh_run(public_ip, args.ssh_user, args.key_file, _READ_FS_BYTES.replace("__MOUNT__", args.mount_point)) + fs_after = int(after_out.strip()) if rc == 0 and after_out.strip().isdigit() else None + result["fs_bytes_after"] = fs_after + + ebs_grew = ec2.describe_volumes(VolumeIds=[args.volume_id])["Volumes"][0]["Size"] == result["new_size_gib"] + fs_grew = fs_before is not None and fs_after is not None and fs_after > fs_before + if ebs_grew and fs_grew: + operations["verify_size"]["passed"] = True + else: + _fail( + operations["verify_size"], + f"Size did not grow as expected (ebs_grew={ebs_grew}, fs_before={fs_before}, fs_after={fs_after})", + ) + + result["success"] = all(op["passed"] for op in operations.values()) + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/aws/scripts/common/ebs.py b/isvctl/configs/providers/aws/scripts/common/ebs.py new file mode 100644 index 00000000..4c5c34be --- /dev/null +++ b/isvctl/configs/providers/aws/scripts/common/ebs.py @@ -0,0 +1,322 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared EBS (block volume) helper utilities. + +Centralizes the boto3 volume / snapshot lifecycle used by the +block-storage validation scripts (create / attach / snapshot / restore / +resize / detach / delete) plus the Nitro NVMe device-path mapping needed +to find an attached volume from inside the guest. + +Device mapping note: on Nitro instances EBS volumes are surfaced as NVMe +devices whose kernel name (``/dev/nvme1n1``) is non-deterministic, but +udev creates a stable by-id symlink that embeds the volume ID with the +dash removed:: + + /dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol0123456789abcdef0 + +Scripts resolve that symlink in-guest (``readlink -f``) rather than +guessing ``/dev/sdf`` vs ``/dev/nvme1n1``. +""" + +from __future__ import annotations + +import sys +import time +from typing import Any + +from botocore.exceptions import ClientError + +# Ownership tag so cleanup and audits can tell suite-created volumes apart +# from anything the account already had. +_ISV_CREATED_BY_TAG = {"Key": "CreatedBy", "Value": "isvtest"} + +# AWS error codes meaning the volume/snapshot is already gone - treated as +# success in best-effort cleanup paths. +_NOT_FOUND_CODES = frozenset({"InvalidVolume.NotFound", "InvalidSnapshot.NotFound"}) + + +def nvme_serial_for_volume(volume_id: str) -> str: + """Return the NVMe serial AWS assigns to an EBS volume (the ID minus dashes).""" + return volume_id.replace("-", "") + + +def guest_by_id_path(volume_id: str) -> str: + """Return the stable ``/dev/disk/by-id`` path for an attached EBS volume on Nitro.""" + return f"/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_{nvme_serial_for_volume(volume_id)}" + + +def _tag_spec(resource_type: str, name: str) -> dict[str, Any]: + """Build a TagSpecifications entry carrying the Name and ownership tags.""" + return { + "ResourceType": resource_type, + "Tags": [{"Key": "Name", "Value": name}, _ISV_CREATED_BY_TAG], + } + + +def create_volume( + ec2: Any, + availability_zone: str, + size_gib: int, + *, + volume_type: str = "gp3", + name: str = "isv-validate-block", +) -> str: + """Create an empty EBS volume in ``availability_zone`` and return its ID.""" + response = ec2.create_volume( + AvailabilityZone=availability_zone, + Size=size_gib, + VolumeType=volume_type, + TagSpecifications=[_tag_spec("volume", name)], + ) + return response["VolumeId"] + + +def create_volume_from_snapshot( + ec2: Any, + snapshot_id: str, + availability_zone: str, + *, + volume_type: str = "gp3", + name: str = "isv-validate-restore", +) -> str: + """Create a volume restored from ``snapshot_id`` and return its ID.""" + response = ec2.create_volume( + SnapshotId=snapshot_id, + AvailabilityZone=availability_zone, + VolumeType=volume_type, + TagSpecifications=[_tag_spec("volume", name)], + ) + return response["VolumeId"] + + +def attach_volume(ec2: Any, volume_id: str, instance_id: str, device: str) -> None: + """Attach ``volume_id`` to ``instance_id`` at the requested block device name.""" + ec2.attach_volume(VolumeId=volume_id, InstanceId=instance_id, Device=device) + + +def detach_volume(ec2: Any, volume_id: str, *, force: bool = False) -> None: + """Detach ``volume_id`` from whatever instance it is attached to.""" + ec2.detach_volume(VolumeId=volume_id, Force=force) + + +def delete_volume(ec2: Any, volume_id: str) -> None: + """Delete ``volume_id`` (must already be detached / available).""" + ec2.delete_volume(VolumeId=volume_id) + + +def create_snapshot( + ec2: Any, + volume_id: str, + *, + description: str = "ISV block-storage validation snapshot", + name: str = "isv-validate-snap", +) -> str: + """Create a point-in-time snapshot of ``volume_id`` and return its ID.""" + response = ec2.create_snapshot( + VolumeId=volume_id, + Description=description, + TagSpecifications=[_tag_spec("snapshot", name)], + ) + return response["SnapshotId"] + + +def delete_snapshot(ec2: Any, snapshot_id: str) -> None: + """Delete ``snapshot_id``.""" + ec2.delete_snapshot(SnapshotId=snapshot_id) + + +def modify_volume_size(ec2: Any, volume_id: str, new_size_gib: int) -> None: + """Request a grow of ``volume_id`` to ``new_size_gib`` via ModifyVolume.""" + ec2.modify_volume(VolumeId=volume_id, Size=new_size_gib) + + +def wait_for_volume_available(ec2: Any, volume_id: str, *, delay: int = 5, max_attempts: int = 60) -> None: + """Block until ``volume_id`` reaches the ``available`` (detached) state.""" + waiter = ec2.get_waiter("volume_available") + waiter.wait(VolumeIds=[volume_id], WaiterConfig={"Delay": delay, "MaxAttempts": max_attempts}) + + +def wait_for_volume_in_use(ec2: Any, volume_id: str, *, delay: int = 5, max_attempts: int = 60) -> None: + """Block until ``volume_id`` reaches the ``in-use`` (attached) state.""" + waiter = ec2.get_waiter("volume_in_use") + waiter.wait(VolumeIds=[volume_id], WaiterConfig={"Delay": delay, "MaxAttempts": max_attempts}) + + +def wait_for_volume_deleted(ec2: Any, volume_id: str, *, delay: int = 5, max_attempts: int = 60) -> None: + """Block until ``volume_id`` is fully deleted.""" + waiter = ec2.get_waiter("volume_deleted") + waiter.wait(VolumeIds=[volume_id], WaiterConfig={"Delay": delay, "MaxAttempts": max_attempts}) + + +def wait_for_snapshot_completed(ec2: Any, snapshot_id: str, *, delay: int = 15, max_attempts: int = 80) -> None: + """Block until ``snapshot_id`` finishes (state ``completed``).""" + waiter = ec2.get_waiter("snapshot_completed") + waiter.wait(SnapshotIds=[snapshot_id], WaiterConfig={"Delay": delay, "MaxAttempts": max_attempts}) + + +def wait_for_modification_complete( + ec2: Any, + volume_id: str, + *, + timeout: int = 900, + interval: int = 15, +) -> str: + """Poll ModifyVolume progress until the new size is usable by the guest. + + A volume modification advances ``modifying -> optimizing -> completed``. + The larger capacity is already visible to the OS once the state reaches + ``optimizing``, so the in-guest grow can proceed without waiting for the + (potentially long) background optimization to finish. + + Args: + ec2: Boto3 EC2 client. + volume_id: Volume being modified. + timeout: Total seconds to wait before giving up. + interval: Seconds between describe calls. + + Returns: + The terminal modification state observed (``optimizing`` or ``completed``). + + Raises: + RuntimeError: If the modification fails or does not reach a usable + state within ``timeout``. + """ + deadline = time.monotonic() + timeout + while True: + response = ec2.describe_volumes_modifications(VolumeIds=[volume_id]) + modifications = response.get("VolumesModifications", []) + state = modifications[0].get("ModificationState") if modifications else None + + if state in ("optimizing", "completed"): + return state + if state == "failed": + status = modifications[0].get("StatusMessage", "") + raise RuntimeError(f"Volume modification failed for {volume_id}: {status}") + + if time.monotonic() >= deadline: + raise RuntimeError(f"Timed out waiting for {volume_id} modification (last state: {state})") + time.sleep(interval) + + +def is_volume_attached_to(ec2: Any, volume_id: str, instance_id: str) -> bool: + """Return True if ``volume_id`` is attached to ``instance_id`` and in-use.""" + response = ec2.describe_volumes(VolumeIds=[volume_id]) + volumes = response.get("Volumes", []) + if not volumes: + return False + volume = volumes[0] + if volume.get("State") != "in-use": + return False + return any(att.get("InstanceId") == instance_id for att in volume.get("Attachments", [])) + + +def detach_and_delete_volume(ec2: Any, volume_id: str) -> str | None: + """Detach (if needed) and delete ``volume_id`` best-effort. + + Returns an error message on failure, or ``None`` if the volume was + deleted or was already gone. Safe to call from ``finally`` blocks. + """ + try: + try: + detach_volume(ec2, volume_id, force=True) + wait_for_volume_available(ec2, volume_id) + except ClientError as e: + code = e.response.get("Error", {}).get("Code", "") + if code in _NOT_FOUND_CODES: + return None + # IncorrectState means it is already detached/available - fall through to delete. + if code != "IncorrectState": + return str(e) + + delete_volume(ec2, volume_id) + wait_for_volume_deleted(ec2, volume_id) + return None + except ClientError as e: + code = e.response.get("Error", {}).get("Code", "") + if code in _NOT_FOUND_CODES: + return None + return str(e) + + +def delete_snapshot_best_effort(ec2: Any, snapshot_id: str) -> str | None: + """Delete ``snapshot_id`` best-effort. Returns an error message or ``None``.""" + try: + delete_snapshot(ec2, snapshot_id) + return None + except ClientError as e: + code = e.response.get("Error", {}).get("Code", "") + if code in _NOT_FOUND_CODES: + return None + return str(e) + + +def wait_for_attachment_device(host: str, user: str, key_file: str, volume_id: str, *, attempts: int = 30) -> bool: + """Poll over SSH until the volume's by-id symlink exists in the guest. + + Returns True once ``guest_by_id_path(volume_id)`` is present, else False. + """ + # Local import to avoid a hard dependency for callers that only need the + # boto3 helpers (and to keep the common package import-light). + from common.ssh_utils import ssh_run + + by_id = guest_by_id_path(volume_id) + for attempt in range(1, attempts + 1): + rc, _, _ = ssh_run(host, user, key_file, f"test -e {by_id}") + if rc == 0: + return True + print(f" Waiting for {by_id} in guest... (attempt {attempt}/{attempts})", file=sys.stderr) + time.sleep(5) + return False + + +# In-guest: resolve the volume's partition via its stable by-id symlink, mount +# it (idempotently), and print the sentinel file contents to stdout. +SENTINEL_FILENAME = "isv-sentinel.txt" +_READ_SENTINEL_SCRIPT = r""" +set -euo pipefail +BYID="__BYID__" +MOUNT="__MOUNT__" +for _ in $(seq 1 30); do [ -e "__BYID__-part1" ] && break; sleep 2; done +PART=$(readlink -f "__BYID__-part1") +sudo mkdir -p "$MOUNT" +mountpoint -q "$MOUNT" || sudo mount "$PART" "$MOUNT" +cat "$MOUNT/__FILENAME__" +""" + + +def mount_and_read_sentinel( + host: str, + user: str, + key_file: str, + volume_id: str, + mount_point: str, + *, + timeout: int = 120, +) -> tuple[int, str, str]: + """Mount ``volume_id`` in the guest and read the sentinel file. + + Returns ``(exit_code, stdout, stderr)`` from the remote command. ``stdout`` + holds the raw sentinel contents on success (callers byte-compare it). + """ + # Local import keeps the common package import-light for boto3-only callers. + from common.ssh_utils import ssh_run + + script = ( + _READ_SENTINEL_SCRIPT.replace("__BYID__", guest_by_id_path(volume_id)) + .replace("__MOUNT__", mount_point) + .replace("__FILENAME__", SENTINEL_FILENAME) + ) + return ssh_run(host, user, key_file, script, timeout=timeout) diff --git a/isvctl/configs/suites/block-storage.yaml b/isvctl/configs/suites/block-storage.yaml new file mode 100644 index 00000000..304a0b7a --- /dev/null +++ b/isvctl/configs/suites/block-storage.yaml @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Block Storage Validation - Canonical Contract +# +# Validations-only contract for persistent block storage (EBS-style +# volumes). Provider configs import this file and supply their own +# commands (steps + scripts). +# +# This file defines WHAT to validate, not HOW to run it. Each validation +# only inspects JSON output fields. As long as your scripts output the +# right fields, the validations pass regardless of which cloud you use. +# +# Coverage notes: +# - A shared fixture launches one instance and creates + attaches a single +# block volume (formatted, mounted, seeded with a sentinel file). All +# three test-phase checks reuse that fixture: +# - snapshot_lifecycle - DATASVC-XX-02 (issue #321): snapshot the volume, +# restore it to a new volume, attach + mount the restore, and verify the +# sentinel data survived the round-trip. +# - volume_resize - DATASVC-XX-03 (issue #322): grow the volume +# (ModifyVolume), then grow the in-guest partition + filesystem and +# confirm the larger size is visible to the guest. +# - volume_persistence - DATASVC-XX-04 (issue #323): stop + start the +# instance and confirm the volume stays attached with its data intact. +# +# Quick start: +# 1. Copy providers/my-isv/config/block-storage.yaml to providers//config/block-storage.yaml +# 2. Replace stub paths with your platform scripts +# 3. Run: uv run isvctl test run -f isvctl/configs/providers//config/block-storage.yaml +# +# See also: +# - my-isv living example: providers/my-isv/config/block-storage.yaml +# - AWS reference: providers/aws/config/block-storage.yaml + providers/aws/scripts/block-storage/ +# - Per-step field breakdown: ../suites/README.md + +version: "1.0" + +tests: + platform: block_storage + cluster_name: "block-storage-validation" + description: "Block storage: snapshots, resizing, and persistence across restarts" + + settings: + # Provider-specific - override in your provider config + region: "" + instance_type: "" + volume_size_gib: "10" + teardown_flag: "{{(env.BLOCK_STORAGE_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" + + # ─── Validations ───────────────────────────────────────────────── + # These are PROVIDER-AGNOSTIC. They only check JSON field names/values. + validations: + # ─── Fixture: instance + attached, formatted, seeded volume ────── + instance_launched: + step: launch_instance + checks: + InstanceStateCheck: + expected_state: "running" + + fixture_volume: + step: create_volume + checks: + StepSuccessCheck: {} + FieldExistsCheck: + fields: ["volume_id", "mount_point", "operations"] + CrudOperationsCheck: + operations: ["create", "attach", "format", "mount", "write_sentinel"] + + # ─── DATASVC-XX-02: volume snapshots (issue #321) ──────────────── + snapshot_lifecycle: + step: snapshot_lifecycle + checks: + StepSuccessCheck: {} + FieldExistsCheck: + fields: ["volume_id", "snapshot_id", "operations"] + CrudOperationsCheck: + operations: ["create_snapshot", "restore_volume", "verify_data"] + + # ─── DATASVC-XX-03: volume resizing (issue #322) ───────────────── + volume_resize: + step: volume_resize + checks: + StepSuccessCheck: {} + FieldExistsCheck: + fields: ["volume_id", "operations"] + CrudOperationsCheck: + operations: ["modify_volume", "grow_partition", "resize_filesystem", "verify_size"] + + # ─── DATASVC-XX-04: persistence across restarts (issue #323) ───── + volume_persistence: + step: volume_persistence + checks: + StepSuccessCheck: {} + FieldExistsCheck: + fields: ["volume_id", "operations"] + CrudOperationsCheck: + operations: ["stop", "start", "verify_attached", "verify_data"] + + # ─── Teardown ──────────────────────────────────────────────────── + teardown_checks: + step: teardown_volume + checks: + StepSuccessCheck: {} + + exclude: + labels: [] From ec5f6b4db5862d6d1a4625e734dac65ac5a0e49c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 28 May 2026 20:02:09 +0000 Subject: [PATCH 02/10] feat(block-storage): add my-isv scaffold and wire demo-test Adds the my-isv living-example scaffold (TODO blocks + ISVCTL_DEMO_MODE dummy success) for the five block-storage scripts and the provider config that imports the suite. Adds block-storage to MY_ISV_DOMAINS so 'make demo-test' exercises the whole pipeline end-to-end with no cloud. Signed-off-by: Cursor Agent Co-authored-by: Alexandre Begnoche --- Makefile | 2 +- .../my-isv/config/block-storage.yaml | 157 ++++++++++++++++++ .../scripts/block-storage/create_volume.py | 117 +++++++++++++ .../block-storage/snapshot_lifecycle.py | 111 +++++++++++++ .../scripts/block-storage/teardown_volume.py | 91 ++++++++++ .../block-storage/volume_persistence.py | 111 +++++++++++++ .../scripts/block-storage/volume_resize.py | 106 ++++++++++++ 7 files changed, 694 insertions(+), 1 deletion(-) create mode 100644 isvctl/configs/providers/my-isv/config/block-storage.yaml create mode 100644 isvctl/configs/providers/my-isv/scripts/block-storage/create_volume.py create mode 100644 isvctl/configs/providers/my-isv/scripts/block-storage/snapshot_lifecycle.py create mode 100644 isvctl/configs/providers/my-isv/scripts/block-storage/teardown_volume.py create mode 100644 isvctl/configs/providers/my-isv/scripts/block-storage/volume_persistence.py create mode 100644 isvctl/configs/providers/my-isv/scripts/block-storage/volume_resize.py diff --git a/Makefile b/Makefile index 4a317d96..ce5ef621 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -MY_ISV_DOMAINS := bare_metal control-plane iam image-registry network observability security vm +MY_ISV_DOMAINS := bare_metal block-storage control-plane iam image-registry network observability security vm DEMO_TARGETS := $(addprefix demo-,$(MY_ISV_DOMAINS)) .PHONY: help pre-commit build test coverage clean lint format install bump-patch bump-fix bump-minor bump-feat bump-major bump bump-check \ diff --git a/isvctl/configs/providers/my-isv/config/block-storage.yaml b/isvctl/configs/providers/my-isv/config/block-storage.yaml new file mode 100644 index 00000000..66748479 --- /dev/null +++ b/isvctl/configs/providers/my-isv/config/block-storage.yaml @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# my-isv Block Storage Validation Configuration - Living Example +# +# Imports validations from the provider-agnostic block-storage template and +# wires the commands to the generic template scripts. Those scripts ship with +# dummy-success values so this example runs end-to-end without any real cloud +# infrastructure (ISVCTL_DEMO_MODE=1, used by `make demo-test`). +# +# The launch_instance / teardown steps reuse the VM scaffold scripts; the +# volume steps live in scripts/block-storage/. +# +# Steps: +# SETUP +# 1. launch_instance - Provision the fixture instance +# 2. create_volume - Create + attach + format + mount + seed a volume +# TEST +# 3. snapshot_lifecycle - DATASVC-XX-02 (#321): snapshot -> restore -> verify data +# 4. volume_resize - DATASVC-XX-03 (#322): grow volume + growpart/resize2fs +# 5. volume_persistence - DATASVC-XX-04 (#323): stop/start -> verify reattach + data +# TEARDOWN +# 6. teardown_volume - Detach + delete the fixture volume +# 7. teardown - Terminate the instance +# +# Usage: +# uv run isvctl test run -f isvctl/configs/providers/my-isv/config/block-storage.yaml + +import: + - ../../../suites/block-storage.yaml + +version: "1.0" + +commands: + block_storage: + phases: ["setup", "test", "teardown"] + steps: + - name: launch_instance + phase: setup + command: "python ../scripts/vm/launch_instance.py" + args: + - "--name" + - "isv-test-block" + - "--instance-type" + - "{{instance_type}}" + - "--region" + - "{{region}}" + timeout: 60 + + - name: create_volume + phase: setup + command: "python ../scripts/block-storage/create_volume.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + - "--size-gib" + - "{{volume_size_gib}}" + timeout: 60 + + - name: snapshot_lifecycle + phase: test + command: "python ../scripts/block-storage/snapshot_lifecycle.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--volume-id" + - "{{steps.create_volume.volume_id}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + - "--expected-content" + - "{{steps.create_volume.sentinel_content}}" + timeout: 60 + + - name: volume_resize + phase: test + command: "python ../scripts/block-storage/volume_resize.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--volume-id" + - "{{steps.create_volume.volume_id}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + - "--mount-point" + - "{{steps.create_volume.mount_point}}" + timeout: 60 + + - name: volume_persistence + phase: test + command: "python ../scripts/block-storage/volume_persistence.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--volume-id" + - "{{steps.create_volume.volume_id}}" + - "--key-file" + - "{{steps.launch_instance.key_file}}" + - "--mount-point" + - "{{steps.create_volume.mount_point}}" + - "--expected-content" + - "{{steps.create_volume.sentinel_content}}" + timeout: 60 + + - name: teardown_volume + phase: teardown + command: "python ../scripts/block-storage/teardown_volume.py" + args: + - "--region" + - "{{region}}" + - "--volume-id" + - "{{steps.create_volume.volume_id}}" + - "{{teardown_flag}}" + timeout: 60 + + - name: teardown + phase: teardown + command: "python ../scripts/vm/teardown.py" + args: + - "--instance-id" + - "{{steps.launch_instance.instance_id}}" + - "--region" + - "{{region}}" + - "--delete-key-pair" + - "--delete-security-group" + timeout: 60 + +tests: + cluster_name: "my-isv-block-storage-validation" + description: "my-isv block storage validation (generic stubs, dummy overrides)" + + settings: + region: "my-isv-region-1" + instance_type: "my-isv.standard.1x" + volume_size_gib: "10" + teardown_flag: "{{(env.BLOCK_STORAGE_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" diff --git a/isvctl/configs/providers/my-isv/scripts/block-storage/create_volume.py b/isvctl/configs/providers/my-isv/scripts/block-storage/create_volume.py new file mode 100644 index 00000000..de7e8707 --- /dev/null +++ b/isvctl/configs/providers/my-isv/scripts/block-storage/create_volume.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Block-storage fixture: create, attach, format, mount, and seed a volume. + +Provider-agnostic template - replace the TODO block with your platform's +block-volume API calls plus in-guest formatting. This single fixture is +shared by the snapshot, resize, and persistence tests, so the volume must +be left attached, formatted, mounted, and seeded with the sentinel file. + +Required JSON output fields: + success (bool) - true iff every operation passed + platform (str) - "block_storage" + test_name (str) - "create_volume" + volume_id (str) - identifier of the created volume (consumed by later steps) + mount_point (str) - where the volume is mounted in the guest + sentinel_content (str) - exact content written to the sentinel file (consumed by later steps) + operations: { + "create": {"passed": bool, "error": str?}, + "attach": {"passed": bool, "error": str?}, + "format": {"passed": bool, "error": str?}, + "mount": {"passed": bool, "error": str?}, + "write_sentinel": {"passed": bool, "error": str?} + } + +Usage: + python create_volume.py --instance-id --region --key-file --size-gib 10 + +Reference implementation (AWS): + ../aws/block-storage/create_volume.py +""" + +import argparse +import json +import os +import sys +import uuid +from typing import Any + +# ISVCTL_DEMO_MODE=1 enables demo-success output (used by `make demo-test`). +DEMO_MODE = os.environ.get("ISVCTL_DEMO_MODE") == "1" + + +def main() -> int: + """Create + attach + format + mount + seed a block volume; print JSON.""" + parser = argparse.ArgumentParser(description="Block-storage fixture: create + seed a volume") + parser.add_argument("--instance-id", default="", help="Instance to attach the volume to") + parser.add_argument("--region", default="", help="Cloud region") + parser.add_argument("--key-file", default="", help="Path to SSH private key") + parser.add_argument("--size-gib", type=int, default=10, help="Volume size in GiB") + parser.add_argument("--mount-point", default="/mnt/isv-block", help="In-guest mount point") + args = parser.parse_args() + + sentinel_content = f"isv-ncp-validate-block-storage-{uuid.uuid4().hex}" + operations: dict[str, dict[str, Any]] = { + "create": {"passed": False}, + "attach": {"passed": False}, + "format": {"passed": False}, + "mount": {"passed": False}, + "write_sentinel": {"passed": False}, + } + result: dict[str, Any] = { + "success": False, + "platform": "block_storage", + "test_name": "create_volume", + "volume_id": "", + "mount_point": args.mount_point, + "size_gib": args.size_gib, + "sentinel_content": sentinel_content, + "operations": operations, + } + + # ╔══════════════════════════════════════════════════════════════════╗ + # ║ TODO: Replace this block with your platform's implementation ║ + # ║ ║ + # ║ 1. CreateVolume(size_gib) in the instance's zone ║ + # ║ -> operations["create"]["passed"] = True ║ + # ║ 2. AttachVolume(volume_id, instance_id) ║ + # ║ -> operations["attach"]["passed"] = True ║ + # ║ 3. Over SSH: partition + mkfs the attached device ║ + # ║ -> operations["format"]["passed"] = True ║ + # ║ 4. Over SSH: mount it at args.mount_point ║ + # ║ -> operations["mount"]["passed"] = True ║ + # ║ 5. Over SSH: write sentinel_content to /isv-sentinel.txt ║ + # ║ -> operations["write_sentinel"]["passed"] = True ║ + # ║ 6. result["volume_id"] = volume_id ║ + # ║ 7. result["success"] = all(op["passed"] for op in operations…) ║ + # ╚══════════════════════════════════════════════════════════════════╝ + + if DEMO_MODE: + result["volume_id"] = f"dummy-vol-{uuid.uuid4().hex[:8]}" + for op in operations.values(): + op["passed"] = True + result["success"] = True + else: + operations["create"]["error"] = "Not implemented" + result["error"] = "Not implemented - replace with your platform's create/attach/format/mount logic" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/my-isv/scripts/block-storage/snapshot_lifecycle.py b/isvctl/configs/providers/my-isv/scripts/block-storage/snapshot_lifecycle.py new file mode 100644 index 00000000..9baed6ba --- /dev/null +++ b/isvctl/configs/providers/my-isv/scripts/block-storage/snapshot_lifecycle.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DATASVC-XX-02: verify volume snapshots (issue #321). + +Provider-agnostic template - replace the TODO block with your platform's +snapshot + restore API calls. Snapshot the fixture volume, restore it to a +new volume, attach + mount that restore, and byte-compare the sentinel file +against the content the fixture wrote. Clean up the restored volume and the +snapshot afterward. + +Required JSON output fields: + success (bool) - true iff every operation passed + platform (str) - "block_storage" + test_name (str) - "snapshot_lifecycle" + volume_id (str) - source (fixture) volume that was snapshotted + snapshot_id (str) - identifier of the created snapshot + operations: { + "create_snapshot": {"passed": bool, "error": str?}, + "restore_volume": {"passed": bool, "error": str?}, + "verify_data": {"passed": bool, "content_matches": bool, "error": str?} + } + +Usage: + python snapshot_lifecycle.py --volume-id --expected-content + +Reference implementation (AWS): + ../aws/block-storage/snapshot_lifecycle.py +""" + +import argparse +import json +import os +import sys +import uuid +from typing import Any + +# ISVCTL_DEMO_MODE=1 enables demo-success output (used by `make demo-test`). +DEMO_MODE = os.environ.get("ISVCTL_DEMO_MODE") == "1" + + +def main() -> int: + """Snapshot, restore, and verify a volume; print JSON result.""" + parser = argparse.ArgumentParser(description="Verify volume snapshots (DATASVC-XX-02)") + parser.add_argument("--instance-id", default="", help="Instance for the restored volume") + parser.add_argument("--region", default="", help="Cloud region") + parser.add_argument("--volume-id", default="", help="Source (fixture) volume to snapshot") + parser.add_argument("--key-file", default="", help="Path to SSH private key") + parser.add_argument("--expected-content", default="", help="Sentinel content written by the fixture") + args = parser.parse_args() + + operations: dict[str, dict[str, Any]] = { + "create_snapshot": {"passed": False}, + "restore_volume": {"passed": False}, + "verify_data": {"passed": False}, + } + result: dict[str, Any] = { + "success": False, + "platform": "block_storage", + "test_name": "snapshot_lifecycle", + "volume_id": args.volume_id, + "snapshot_id": "", + "operations": operations, + } + + # ╔══════════════════════════════════════════════════════════════════╗ + # ║ TODO: Replace this block with your platform's implementation ║ + # ║ ║ + # ║ 1. CreateSnapshot(volume_id); wait until complete ║ + # ║ -> operations["create_snapshot"]["passed"] = True ║ + # ║ 2. CreateVolume(from snapshot); attach to the instance ║ + # ║ -> operations["restore_volume"]["passed"] = True ║ + # ║ 3. Over SSH: mount the restore, read /isv-sentinel.txt ║ + # ║ matches = (content == args.expected_content) ║ + # ║ -> operations["verify_data"]["content_matches"] = matches ║ + # ║ -> operations["verify_data"]["passed"] = matches ║ + # ║ 4. Best-effort: delete the restored volume and the snapshot ║ + # ║ 5. result["success"] = all(op["passed"] for op in operations…) ║ + # ╚══════════════════════════════════════════════════════════════════╝ + + if DEMO_MODE: + result["snapshot_id"] = f"dummy-snap-{uuid.uuid4().hex[:8]}" + result["restored_volume_id"] = f"dummy-vol-{uuid.uuid4().hex[:8]}" + operations["create_snapshot"]["passed"] = True + operations["restore_volume"]["passed"] = True + operations["verify_data"]["passed"] = True + operations["verify_data"]["content_matches"] = True + result["success"] = True + else: + operations["create_snapshot"]["error"] = "Not implemented" + result["error"] = "Not implemented - replace with your platform's snapshot/restore logic" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/my-isv/scripts/block-storage/teardown_volume.py b/isvctl/configs/providers/my-isv/scripts/block-storage/teardown_volume.py new file mode 100644 index 00000000..04607534 --- /dev/null +++ b/isvctl/configs/providers/my-isv/scripts/block-storage/teardown_volume.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Detach and delete the block-storage fixture volume (teardown). + +Provider-agnostic template - replace the TODO block with your platform's +detach + delete API calls for the fixture volume created by +``create_volume.py``. + +Required JSON output fields: + success (bool) - whether cleanup succeeded + platform (str) - "block_storage" + test_name (str) - "teardown_volume" + resources_deleted (list) - identifiers of volumes that were deleted + message (str) - human-readable summary + +Usage: + python teardown_volume.py --volume-id --region + +Reference implementation (AWS): + ../aws/block-storage/teardown_volume.py +""" + +import argparse +import json +import os +import sys +from typing import Any + +# ISVCTL_DEMO_MODE=1 enables demo-success output (used by `make demo-test`). +DEMO_MODE = os.environ.get("ISVCTL_DEMO_MODE") == "1" + + +def main() -> int: + """Detach and delete the fixture volume; print JSON result.""" + parser = argparse.ArgumentParser(description="Teardown block-storage fixture volume") + parser.add_argument("--region", default="", help="Cloud region") + parser.add_argument("--volume-id", default="", help="Fixture volume to delete") + parser.add_argument("--skip-destroy", action="store_true", help="Skip actual destroy") + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "block_storage", + "test_name": "teardown_volume", + "resources_deleted": [], + "message": "", + } + + if args.skip_destroy: + result["success"] = True + result["message"] = f"Volume {args.volume_id} preserved (--skip-destroy)" + print(json.dumps(result, indent=2)) + return 0 + + # ╔══════════════════════════════════════════════════════════════════╗ + # ║ TODO: Replace this block with your platform's implementation ║ + # ║ ║ + # ║ 1. Detach the volume from the instance (if still attached) ║ + # ║ 2. Delete the volume ║ + # ║ result["resources_deleted"].append(args.volume_id) ║ + # ║ 3. result["success"] = True ║ + # ╚══════════════════════════════════════════════════════════════════╝ + + if DEMO_MODE: + if args.volume_id: + result["resources_deleted"].append(args.volume_id) + result["message"] = "Fixture volume deleted" + result["success"] = True + else: + result["error"] = "Not implemented - replace with your platform's detach + delete logic" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/my-isv/scripts/block-storage/volume_persistence.py b/isvctl/configs/providers/my-isv/scripts/block-storage/volume_persistence.py new file mode 100644 index 00000000..f91a8eb9 --- /dev/null +++ b/isvctl/configs/providers/my-isv/scripts/block-storage/volume_persistence.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DATASVC-XX-04: persistent volumes survive instance restarts (issue #323). + +Provider-agnostic template - replace the TODO block with your platform's +stop/start API calls. Stop and start the instance, then confirm the block +volume is still attached and its sentinel data is intact (the volume must be +re-mounted after the restart). + +Required JSON output fields: + success (bool) - true iff every operation passed + platform (str) - "block_storage" + test_name (str) - "volume_persistence" + volume_id (str) - volume expected to persist + operations: { + "stop": {"passed": bool, "error": str?}, + "start": {"passed": bool, "error": str?}, + "verify_attached": {"passed": bool, "error": str?}, + "verify_data": {"passed": bool, "content_matches": bool, "error": str?} + } + +Usage: + python volume_persistence.py --instance-id --volume-id --expected-content + +Reference implementation (AWS): + ../aws/block-storage/volume_persistence.py +""" + +import argparse +import json +import os +import sys +from typing import Any + +# ISVCTL_DEMO_MODE=1 enables demo-success output (used by `make demo-test`). +DEMO_MODE = os.environ.get("ISVCTL_DEMO_MODE") == "1" + + +def main() -> int: + """Stop/start the instance and verify the volume + data survive; print JSON.""" + parser = argparse.ArgumentParser(description="Verify block volume persistence across restart (DATASVC-XX-04)") + parser.add_argument("--instance-id", default="", help="Instance to restart") + parser.add_argument("--region", default="", help="Cloud region") + parser.add_argument("--volume-id", default="", help="Volume expected to persist") + parser.add_argument("--key-file", default="", help="Path to SSH private key") + parser.add_argument("--mount-point", default="/mnt/isv-block", help="In-guest mount point") + parser.add_argument("--expected-content", default="", help="Sentinel content written by the fixture") + args = parser.parse_args() + + operations: dict[str, dict[str, Any]] = { + "stop": {"passed": False}, + "start": {"passed": False}, + "verify_attached": {"passed": False}, + "verify_data": {"passed": False}, + } + result: dict[str, Any] = { + "success": False, + "platform": "block_storage", + "test_name": "volume_persistence", + "volume_id": args.volume_id, + "instance_id": args.instance_id, + "operations": operations, + } + + # ╔══════════════════════════════════════════════════════════════════╗ + # ║ TODO: Replace this block with your platform's implementation ║ + # ║ ║ + # ║ 1. Stop the instance; wait for it to reach stopped ║ + # ║ -> operations["stop"]["passed"] = True ║ + # ║ 2. Start the instance; wait for it to be reachable over SSH ║ + # ║ -> operations["start"]["passed"] = True ║ + # ║ 3. Confirm the volume is still attached to the instance ║ + # ║ -> operations["verify_attached"]["passed"] = True ║ + # ║ 4. Over SSH: re-mount the volume, read /isv-sentinel.txt ║ + # ║ matches = (content == args.expected_content) ║ + # ║ -> operations["verify_data"]["content_matches"] = matches ║ + # ║ -> operations["verify_data"]["passed"] = matches ║ + # ║ 5. result["success"] = all(op["passed"] for op in operations…) ║ + # ╚══════════════════════════════════════════════════════════════════╝ + + if DEMO_MODE: + operations["stop"]["passed"] = True + operations["start"]["passed"] = True + operations["verify_attached"]["passed"] = True + operations["verify_data"]["passed"] = True + operations["verify_data"]["content_matches"] = True + result["success"] = True + else: + operations["stop"]["error"] = "Not implemented" + result["error"] = "Not implemented - replace with your platform's stop/start + verify logic" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/my-isv/scripts/block-storage/volume_resize.py b/isvctl/configs/providers/my-isv/scripts/block-storage/volume_resize.py new file mode 100644 index 00000000..a5b3f0c4 --- /dev/null +++ b/isvctl/configs/providers/my-isv/scripts/block-storage/volume_resize.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DATASVC-XX-03: verify volume resizing (issue #322). + +Provider-agnostic template - replace the TODO block with your platform's +volume-grow API call plus the in-guest partition + filesystem grow. Confirm +the larger capacity is visible to the guest after the resize. + +Required JSON output fields: + success (bool) - true iff every operation passed + platform (str) - "block_storage" + test_name (str) - "volume_resize" + volume_id (str) - volume that was resized + operations: { + "modify_volume": {"passed": bool, "error": str?}, + "grow_partition": {"passed": bool, "error": str?}, + "resize_filesystem": {"passed": bool, "error": str?}, + "verify_size": {"passed": bool, "error": str?} + } + +Usage: + python volume_resize.py --volume-id --mount-point /mnt/isv-block + +Reference implementation (AWS): + ../aws/block-storage/volume_resize.py +""" + +import argparse +import json +import os +import sys +from typing import Any + +# ISVCTL_DEMO_MODE=1 enables demo-success output (used by `make demo-test`). +DEMO_MODE = os.environ.get("ISVCTL_DEMO_MODE") == "1" + + +def main() -> int: + """Grow the volume + in-guest filesystem and verify; print JSON result.""" + parser = argparse.ArgumentParser(description="Verify volume resizing (DATASVC-XX-03)") + parser.add_argument("--instance-id", default="", help="Instance the volume is attached to") + parser.add_argument("--region", default="", help="Cloud region") + parser.add_argument("--volume-id", default="", help="Volume to resize (fixture volume)") + parser.add_argument("--key-file", default="", help="Path to SSH private key") + parser.add_argument("--mount-point", default="/mnt/isv-block", help="In-guest mount point") + parser.add_argument("--grow-gib", type=int, default=5, help="GiB to add to the volume") + args = parser.parse_args() + + operations: dict[str, dict[str, Any]] = { + "modify_volume": {"passed": False}, + "grow_partition": {"passed": False}, + "resize_filesystem": {"passed": False}, + "verify_size": {"passed": False}, + } + result: dict[str, Any] = { + "success": False, + "platform": "block_storage", + "test_name": "volume_resize", + "volume_id": args.volume_id, + "operations": operations, + } + + # ╔══════════════════════════════════════════════════════════════════╗ + # ║ TODO: Replace this block with your platform's implementation ║ + # ║ ║ + # ║ 1. Read the current volume size, then grow it by args.grow_gib ║ + # ║ -> operations["modify_volume"]["passed"] = True ║ + # ║ 2. Over SSH: growpart the partition ║ + # ║ -> operations["grow_partition"]["passed"] = True ║ + # ║ 3. Over SSH: resize2fs (or equivalent) the filesystem ║ + # ║ -> operations["resize_filesystem"]["passed"] = True ║ + # ║ 4. Confirm the guest sees the larger filesystem size ║ + # ║ -> operations["verify_size"]["passed"] = True ║ + # ║ 5. result["success"] = all(op["passed"] for op in operations…) ║ + # ╚══════════════════════════════════════════════════════════════════╝ + + if DEMO_MODE: + result["old_size_gib"] = 10 + result["new_size_gib"] = 10 + args.grow_gib + for op in operations.values(): + op["passed"] = True + result["success"] = True + else: + operations["modify_volume"]["error"] = "Not implemented" + result["error"] = "Not implemented - replace with your platform's resize + growpart/resize2fs logic" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) From b640244eead20478cd21bc893f68740e8eb8b1ca Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 28 May 2026 20:02:09 +0000 Subject: [PATCH 03/10] docs(block-storage): document the block-storage suite Adds the block-storage suite to the contracts list and per-step field table in suites/README.md, and a block-storage/ row to the my-isv scaffold domains table. Signed-off-by: Cursor Agent Co-authored-by: Alexandre Begnoche --- .../configs/providers/my-isv/scripts/README.md | 1 + isvctl/configs/suites/README.md | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/isvctl/configs/providers/my-isv/scripts/README.md b/isvctl/configs/providers/my-isv/scripts/README.md index b4e1ed07..4bbae0eb 100644 --- a/isvctl/configs/providers/my-isv/scripts/README.md +++ b/isvctl/configs/providers/my-isv/scripts/README.md @@ -31,6 +31,7 @@ template, then fill in the TODOs. | `control-plane/` | 10 | [`suites/control-plane.yaml`](../../../suites/control-plane.yaml) | [`config/control-plane.yaml`](../config/control-plane.yaml) | [`providers/aws/scripts/control-plane/`](../../aws/scripts/control-plane/) | | `vm/` | 9 | [`suites/vm.yaml`](../../../suites/vm.yaml) | [`config/vm.yaml`](../config/vm.yaml) | [`providers/aws/scripts/vm/`](../../aws/scripts/vm/) | | `bare_metal/` | 12 | [`suites/bare_metal.yaml`](../../../suites/bare_metal.yaml) | [`config/bare_metal.yaml`](../config/bare_metal.yaml) | [`providers/aws/scripts/bare_metal/`](../../aws/scripts/bare_metal/) | +| `block-storage/` | 5 | [`suites/block-storage.yaml`](../../../suites/block-storage.yaml) | [`config/block-storage.yaml`](../config/block-storage.yaml) | [`providers/aws/scripts/block-storage/`](../../aws/scripts/block-storage/) | | `network/` | 18 | [`suites/network.yaml`](../../../suites/network.yaml) | [`config/network.yaml`](../config/network.yaml) | [`providers/aws/scripts/network/`](../../aws/scripts/network/) | | `observability/` | 1 | [`suites/observability.yaml`](../../../suites/observability.yaml) | [`config/observability.yaml`](../config/observability.yaml) | [`providers/aws/scripts/observability/`](../../aws/scripts/observability/) | | `image-registry/` | 7 | [`suites/image-registry.yaml`](../../../suites/image-registry.yaml) | [`config/image-registry.yaml`](../config/image-registry.yaml) | [`providers/aws/scripts/image-registry/`](../../aws/scripts/image-registry/) | diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index 216752b8..ed4718e5 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -14,6 +14,7 @@ Suites: [`network`](network.yaml), [`vm`](vm.yaml), [`bare_metal`](bare_metal.yaml), +[`block-storage`](block-storage.yaml), [`observability`](observability.yaml), [`k8s`](k8s.yaml), [`slurm`](slurm.yaml), @@ -121,6 +122,22 @@ For the domain / script-count / AWS-reference overview see the | `query_sanitization` | test | `providers/nico/scripts/sanitization/query_sanitization.py` | `machines_checked`, `machines[].{available,in_use,has_gpu,served_tenant,sanitized,stale_tenant_binding,vendor,product_name,bios_version,transitions}` | | `query_attestation` | test | `providers/nico/scripts/attestation/query_attestation.py` | `machines_checked`, `machines[].{attestation_supported,nonce_verified,attestation_signature_valid,secure_boot_enabled,boot_measurements_attested,measured_boot_state}` | +### Block Storage (`block-storage.yaml`) + +A shared fixture (`launch_instance` + `create_volume`) provisions one instance with a +single attached, formatted, mounted, and seeded block volume. The three test-phase +steps (DATASVC-XX-02/03/04) all reuse that fixture. + +| Step | Phase | Script | Key JSON Fields | +|------|-------|--------|-----------------| +| `launch_instance` | setup | `providers/my-isv/scripts/vm/launch_instance.py` | `instance_id`, `state`, `public_ip`, `key_file` (reuses VM script) | +| `create_volume` | setup | `providers/my-isv/scripts/block-storage/create_volume.py` | `volume_id`, `mount_point`, `sentinel_content`, `operations.{create,attach,format,mount,write_sentinel}` | +| `snapshot_lifecycle` | test | `providers/my-isv/scripts/block-storage/snapshot_lifecycle.py` | `volume_id`, `snapshot_id`, `operations.{create_snapshot,restore_volume,verify_data}` (verify_data includes `content_matches`) | +| `volume_resize` | test | `providers/my-isv/scripts/block-storage/volume_resize.py` | `volume_id`, `operations.{modify_volume,grow_partition,resize_filesystem,verify_size}` | +| `volume_persistence` | test | `providers/my-isv/scripts/block-storage/volume_persistence.py` | `volume_id`, `operations.{stop,start,verify_attached,verify_data}` (verify_data includes `content_matches`) | +| `teardown_volume` | teardown | `providers/my-isv/scripts/block-storage/teardown_volume.py` | `resources_deleted`, `message` | +| `teardown` | teardown | `providers/my-isv/scripts/vm/teardown.py` | `resources_deleted`, `message` (reuses VM script) | + ### Kubernetes (`k8s.yaml`) | Step | Phase | Script | From a9d94957894c06b04ad13f23b12b759db95808f2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 28 May 2026 20:02:09 +0000 Subject: [PATCH 04/10] test(block-storage): add AWS block-storage script unit tests Covers the common/ebs Nitro device mapping and modification waiter, plus happy-path and failure cases (guest setup failure, snapshot content mismatch, cleanup leak, growpart failure, volume detached after restart) for the five AWS block-storage scripts using boto3/ssh fakes. Signed-off-by: Cursor Agent Co-authored-by: Alexandre Begnoche --- .../tests/test_aws_block_storage_scripts.py | 502 ++++++++++++++++++ 1 file changed, 502 insertions(+) create mode 100644 isvctl/tests/test_aws_block_storage_scripts.py diff --git a/isvctl/tests/test_aws_block_storage_scripts.py b/isvctl/tests/test_aws_block_storage_scripts.py new file mode 100644 index 00000000..1f7d007b --- /dev/null +++ b/isvctl/tests/test_aws_block_storage_scripts.py @@ -0,0 +1,502 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for AWS block-storage reference scripts (DATASVC-XX-02/03/04).""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest +from botocore.exceptions import ClientError + +ISVCTL_ROOT = Path(__file__).resolve().parents[1] +AWS_BLOCK_STORAGE_SCRIPTS = ISVCTL_ROOT / "configs" / "providers" / "aws" / "scripts" / "block-storage" + +EXPECTED_CONTENT = "isv-ncp-validate-block-storage-deadbeef" + + +def _load_script(script_name: str) -> ModuleType: + """Load an AWS block-storage script as a module for direct testing.""" + script_path = AWS_BLOCK_STORAGE_SCRIPTS / script_name + spec = importlib.util.spec_from_file_location(f"test_bs_{script_path.stem}", script_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _client_error(code: str) -> ClientError: + """Build a ClientError with the given AWS error code.""" + return ClientError({"Error": {"Code": code, "Message": code}}, "Op") + + +class FakeWaiter: + """No-op boto3 waiter.""" + + def wait(self, **kwargs: Any) -> None: + """Return immediately - nothing to wait for in tests.""" + + +class FakeEc2: + """Minimal fake EC2 client covering the raw calls the scripts make.""" + + def __init__( + self, + *, + availability_zone: str = "us-west-2a", + public_ip: str | None = "203.0.113.10", + volume_size: int = 10, + attached_instance: str = "i-fixture", + ) -> None: + """Seed instance/volume describe responses used by the scripts.""" + self.availability_zone = availability_zone + self.public_ip = public_ip + self.size = volume_size + self.attached_instance = attached_instance + self.stopped: list[str] = [] + self.started: list[str] = [] + + def describe_instances(self, InstanceIds: list[str]) -> dict[str, Any]: + """Return a single instance with AZ + public IP.""" + return { + "Reservations": [ + { + "Instances": [ + { + "Placement": {"AvailabilityZone": self.availability_zone}, + "PublicIpAddress": self.public_ip, + } + ] + } + ] + } + + def describe_volumes(self, VolumeIds: list[str]) -> dict[str, Any]: + """Return the (mutable) size and an in-use attachment.""" + return { + "Volumes": [ + { + "Size": self.size, + "State": "in-use", + "Attachments": [{"InstanceId": self.attached_instance}], + } + ] + } + + def modify_volume(self, VolumeId: str, Size: int) -> dict[str, Any]: + """Apply the new size so a later describe reflects the grow.""" + self.size = Size + return {"VolumeModification": {"ModificationState": "modifying"}} + + def stop_instances(self, InstanceIds: list[str]) -> None: + """Record the stop call.""" + self.stopped.extend(InstanceIds) + + def start_instances(self, InstanceIds: list[str]) -> None: + """Record the start call.""" + self.started.extend(InstanceIds) + + def get_waiter(self, name: str) -> FakeWaiter: + """Return a no-op waiter.""" + return FakeWaiter() + + +def _patch_ebs_volume_ops(monkeypatch: pytest.MonkeyPatch, module: ModuleType) -> None: + """Patch the ebs boto3 wrappers to fast, side-effect-free fakes.""" + monkeypatch.setattr(module.ebs, "create_volume", lambda *a, **k: "vol-fixture") + monkeypatch.setattr(module.ebs, "create_volume_from_snapshot", lambda *a, **k: "vol-restore") + monkeypatch.setattr(module.ebs, "attach_volume", lambda *a, **k: None) + monkeypatch.setattr(module.ebs, "wait_for_volume_available", lambda *a, **k: None) + monkeypatch.setattr(module.ebs, "wait_for_volume_in_use", lambda *a, **k: None) + monkeypatch.setattr(module.ebs, "wait_for_attachment_device", lambda *a, **k: True) + + +# --------------------------------------------------------------------------- +# common/ebs.py pure helpers +# --------------------------------------------------------------------------- + + +def test_nvme_serial_and_by_id_path_drop_dash() -> None: + """The Nitro NVMe serial / by-id path embed the volume ID minus the dash.""" + module = _load_script("create_volume.py") + assert module.ebs.nvme_serial_for_volume("vol-0123456789abcdef0") == "vol0123456789abcdef0" + assert module.ebs.guest_by_id_path("vol-0123456789abcdef0") == ( + "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol0123456789abcdef0" + ) + + +def test_detach_and_delete_volume_treats_missing_as_gone() -> None: + """A volume that is already gone is a successful cleanup (returns None).""" + module = _load_script("teardown_volume.py") + + class GoneEc2: + def detach_volume(self, **kwargs: Any) -> None: + raise _client_error("InvalidVolume.NotFound") + + assert module.ebs.detach_and_delete_volume(GoneEc2(), "vol-x") is None + + +def test_wait_for_modification_complete_returns_usable_state() -> None: + """The modification wait returns once the new size is usable (optimizing).""" + module = _load_script("volume_resize.py") + + class ModEc2: + def describe_volumes_modifications(self, VolumeIds: list[str]) -> dict[str, Any]: + return {"VolumesModifications": [{"ModificationState": "optimizing"}]} + + assert module.ebs.wait_for_modification_complete(ModEc2(), "vol-x") == "optimizing" + + +def test_wait_for_modification_complete_raises_on_failure() -> None: + """A failed modification raises rather than looping until timeout.""" + module = _load_script("volume_resize.py") + + class FailedEc2: + def describe_volumes_modifications(self, VolumeIds: list[str]) -> dict[str, Any]: + return {"VolumesModifications": [{"ModificationState": "failed", "StatusMessage": "nope"}]} + + with pytest.raises(RuntimeError, match="modification failed"): + module.ebs.wait_for_modification_complete(FailedEc2(), "vol-x") + + +# --------------------------------------------------------------------------- +# create_volume.py +# --------------------------------------------------------------------------- + + +def test_create_volume_happy_path(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """A clean create/attach/format/mount/seed flow passes every operation.""" + module = _load_script("create_volume.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2()) + _patch_ebs_volume_ops(monkeypatch, module) + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + monkeypatch.setattr(module, "ssh_run", lambda *a, **k: (0, "", "")) + monkeypatch.setattr(sys, "argv", ["create_volume.py", "--instance-id", "i-fixture", "--key-file", "/tmp/k.pem"]) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["success"] is True + assert payload["volume_id"] == "vol-fixture" + assert all(op["passed"] for op in payload["operations"].values()) + + +def test_create_volume_guest_setup_failure_fails_step( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A non-zero in-guest setup makes the fixture step fail.""" + module = _load_script("create_volume.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2()) + _patch_ebs_volume_ops(monkeypatch, module) + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + monkeypatch.setattr(module, "ssh_run", lambda *a, **k: (1, "", "mkfs failed")) + monkeypatch.setattr(sys, "argv", ["create_volume.py", "--instance-id", "i-fixture", "--key-file", "/tmp/k.pem"]) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert payload["success"] is False + assert payload["operations"]["format"]["passed"] is False + + +# --------------------------------------------------------------------------- +# snapshot_lifecycle.py +# --------------------------------------------------------------------------- + + +def test_snapshot_lifecycle_happy_path( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A matching restored sentinel passes the snapshot round-trip.""" + module = _load_script("snapshot_lifecycle.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2()) + _patch_ebs_volume_ops(monkeypatch, module) + monkeypatch.setattr(module.ebs, "create_snapshot", lambda *a, **k: "snap-1") + monkeypatch.setattr(module.ebs, "wait_for_snapshot_completed", lambda *a, **k: None) + monkeypatch.setattr(module.ebs, "mount_and_read_sentinel", lambda *a, **k: (0, EXPECTED_CONTENT, "")) + monkeypatch.setattr(module.ebs, "detach_and_delete_volume", lambda *a, **k: None) + monkeypatch.setattr(module.ebs, "delete_snapshot_best_effort", lambda *a, **k: None) + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + monkeypatch.setattr(module, "ssh_run", lambda *a, **k: (0, "", "")) + monkeypatch.setattr( + sys, + "argv", + [ + "snapshot_lifecycle.py", + "--instance-id", + "i-fixture", + "--volume-id", + "vol-fixture", + "--key-file", + "/tmp/k.pem", + "--expected-content", + EXPECTED_CONTENT, + ], + ) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["success"] is True + assert payload["snapshot_id"] == "snap-1" + assert payload["operations"]["verify_data"]["content_matches"] is True + + +def test_snapshot_lifecycle_content_mismatch_fails( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A corrupted restore (sentinel mismatch) fails verify_data.""" + module = _load_script("snapshot_lifecycle.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2()) + _patch_ebs_volume_ops(monkeypatch, module) + monkeypatch.setattr(module.ebs, "create_snapshot", lambda *a, **k: "snap-1") + monkeypatch.setattr(module.ebs, "wait_for_snapshot_completed", lambda *a, **k: None) + monkeypatch.setattr(module.ebs, "mount_and_read_sentinel", lambda *a, **k: (0, "CORRUPT", "")) + monkeypatch.setattr(module.ebs, "detach_and_delete_volume", lambda *a, **k: None) + monkeypatch.setattr(module.ebs, "delete_snapshot_best_effort", lambda *a, **k: None) + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + monkeypatch.setattr(module, "ssh_run", lambda *a, **k: (0, "", "")) + monkeypatch.setattr( + sys, + "argv", + [ + "snapshot_lifecycle.py", + "--instance-id", + "i-fixture", + "--volume-id", + "vol-fixture", + "--key-file", + "/tmp/k.pem", + "--expected-content", + EXPECTED_CONTENT, + ], + ) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert payload["success"] is False + assert payload["operations"]["verify_data"]["content_matches"] is False + + +def test_snapshot_lifecycle_cleanup_error_fails_step( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A leaked restore volume / snapshot makes the snapshot step fail.""" + module = _load_script("snapshot_lifecycle.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2()) + _patch_ebs_volume_ops(monkeypatch, module) + monkeypatch.setattr(module.ebs, "create_snapshot", lambda *a, **k: "snap-1") + monkeypatch.setattr(module.ebs, "wait_for_snapshot_completed", lambda *a, **k: None) + monkeypatch.setattr(module.ebs, "mount_and_read_sentinel", lambda *a, **k: (0, EXPECTED_CONTENT, "")) + monkeypatch.setattr(module.ebs, "detach_and_delete_volume", lambda *a, **k: "DeleteVolume failed") + monkeypatch.setattr(module.ebs, "delete_snapshot_best_effort", lambda *a, **k: None) + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + monkeypatch.setattr(module, "ssh_run", lambda *a, **k: (0, "", "")) + monkeypatch.setattr( + sys, + "argv", + [ + "snapshot_lifecycle.py", + "--instance-id", + "i-fixture", + "--volume-id", + "vol-fixture", + "--key-file", + "/tmp/k.pem", + "--expected-content", + EXPECTED_CONTENT, + ], + ) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert payload["success"] is False + assert payload["cleanup_errors"] == ["DeleteVolume failed"] + + +# --------------------------------------------------------------------------- +# volume_resize.py +# --------------------------------------------------------------------------- + + +def test_volume_resize_happy_path(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """ModifyVolume + growpart + resize2fs + a larger filesystem all pass.""" + module = _load_script("volume_resize.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2(volume_size=10)) + monkeypatch.setattr(module.ebs, "wait_for_modification_complete", lambda *a, **k: "completed") + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + + sizes = iter(["10000000000", "15000000000"]) + + def fake_ssh(host: str, user: str, key: str, script: str, **kwargs: Any) -> tuple[int, str, str]: + if "df -B1" in script: + return (0, next(sizes), "") + return (0, "", "") + + monkeypatch.setattr(module, "ssh_run", fake_ssh) + monkeypatch.setattr(sys, "argv", ["volume_resize.py", "--instance-id", "i-fixture", "--volume-id", "vol-fixture", "--key-file", "/tmp/k.pem"]) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["success"] is True + assert payload["new_size_gib"] == 15 + assert all(op["passed"] for op in payload["operations"].values()) + + +def test_volume_resize_growpart_failure_fails_step( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A failing growpart fails the resize and skips resize2fs.""" + module = _load_script("volume_resize.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2(volume_size=10)) + monkeypatch.setattr(module.ebs, "wait_for_modification_complete", lambda *a, **k: "completed") + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + + def fake_ssh(host: str, user: str, key: str, script: str, **kwargs: Any) -> tuple[int, str, str]: + if "df -B1" in script: + return (0, "10000000000", "") + if "growpart" in script: + return (1, "", "growpart boom") + return (0, "", "") + + monkeypatch.setattr(module, "ssh_run", fake_ssh) + monkeypatch.setattr(sys, "argv", ["volume_resize.py", "--instance-id", "i-fixture", "--volume-id", "vol-fixture", "--key-file", "/tmp/k.pem"]) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert payload["success"] is False + assert payload["operations"]["grow_partition"]["passed"] is False + assert payload["operations"]["resize_filesystem"]["passed"] is False + + +# --------------------------------------------------------------------------- +# volume_persistence.py +# --------------------------------------------------------------------------- + + +def test_volume_persistence_happy_path(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """Stop/start with the volume still attached and data intact passes.""" + module = _load_script("volume_persistence.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2(attached_instance="i-fixture")) + monkeypatch.setattr(module.ebs, "is_volume_attached_to", lambda *a, **k: True) + monkeypatch.setattr(module.ebs, "mount_and_read_sentinel", lambda *a, **k: (0, EXPECTED_CONTENT, "")) + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + monkeypatch.setattr(module, "wait_for_public_ip", lambda *a, **k: "203.0.113.10") + monkeypatch.setattr( + sys, + "argv", + [ + "volume_persistence.py", + "--instance-id", + "i-fixture", + "--volume-id", + "vol-fixture", + "--key-file", + "/tmp/k.pem", + "--expected-content", + EXPECTED_CONTENT, + ], + ) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["success"] is True + assert all(op["passed"] for op in payload["operations"].values()) + + +def test_volume_persistence_detached_after_restart_fails( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A volume that does not reattach after restart fails verify_attached.""" + module = _load_script("volume_persistence.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2()) + monkeypatch.setattr(module.ebs, "is_volume_attached_to", lambda *a, **k: False) + monkeypatch.setattr(module.ebs, "mount_and_read_sentinel", lambda *a, **k: (0, EXPECTED_CONTENT, "")) + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + monkeypatch.setattr(module, "wait_for_public_ip", lambda *a, **k: "203.0.113.10") + monkeypatch.setattr( + sys, + "argv", + [ + "volume_persistence.py", + "--instance-id", + "i-fixture", + "--volume-id", + "vol-fixture", + "--key-file", + "/tmp/k.pem", + "--expected-content", + EXPECTED_CONTENT, + ], + ) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert payload["success"] is False + assert payload["operations"]["verify_attached"]["passed"] is False + + +# --------------------------------------------------------------------------- +# teardown_volume.py +# --------------------------------------------------------------------------- + + +def test_teardown_volume_deletes(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """A successful detach + delete reports the volume as deleted.""" + module = _load_script("teardown_volume.py") + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2()) + monkeypatch.setattr(module.ebs, "detach_and_delete_volume", lambda *a, **k: None) + monkeypatch.setattr(sys, "argv", ["teardown_volume.py", "--volume-id", "vol-fixture"]) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["success"] is True + assert payload["resources_deleted"] == ["vol-fixture"] + + +def test_teardown_volume_skip_destroy(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """--skip-destroy preserves the volume and short-circuits cleanup.""" + module = _load_script("teardown_volume.py") + monkeypatch.setattr(sys, "argv", ["teardown_volume.py", "--volume-id", "vol-fixture", "--skip-destroy"]) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["success"] is True + assert payload["resources_deleted"] == [] From 6f41862a001ddc3f5a96804b3d50cdac6fc9a7cf Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 28 May 2026 20:07:55 +0000 Subject: [PATCH 05/10] fix(block-storage): catch BotoCoreError so waiter failures emit JSON boto3 waiters raise WaiterError (a BotoCoreError, not a ClientError), so a waiter timeout would have propagated uncaught and skipped the JSON contract the validations depend on - the same failure mode #438 fixed for S3. Broaden the AWS script except clauses to (ClientError, BotoCoreError) and make the ebs cleanup helpers swallow BotoCoreError so best-effort teardown never raises out of a finally block. Also pass the snapshot ID to cleanup when CreateSnapshot succeeded but the completion wait failed. Signed-off-by: Cursor Agent Co-authored-by: Alexandre Begnoche --- .../aws/scripts/block-storage/create_volume.py | 4 ++-- .../aws/scripts/block-storage/snapshot_lifecycle.py | 12 +++++++----- .../aws/scripts/block-storage/volume_persistence.py | 4 ++-- .../aws/scripts/block-storage/volume_resize.py | 6 +++--- isvctl/configs/providers/aws/scripts/common/ebs.py | 8 +++++++- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/isvctl/configs/providers/aws/scripts/block-storage/create_volume.py b/isvctl/configs/providers/aws/scripts/block-storage/create_volume.py index 6aaccb34..0ea8addd 100644 --- a/isvctl/configs/providers/aws/scripts/block-storage/create_volume.py +++ b/isvctl/configs/providers/aws/scripts/block-storage/create_volume.py @@ -54,7 +54,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # providers/aws/scripts/ (for common.*) import boto3 -from botocore.exceptions import ClientError, NoCredentialsError +from botocore.exceptions import BotoCoreError, ClientError from common import ebs from common.ssh_utils import ssh_run, wait_for_ssh @@ -148,7 +148,7 @@ def main() -> int: ebs.attach_volume(ec2, volume_id, args.instance_id, args.device) ebs.wait_for_volume_in_use(ec2, volume_id) operations["attach"]["passed"] = True - except (ClientError, NoCredentialsError) as e: + except (ClientError, BotoCoreError) as e: result["error"] = f"Volume create/attach failed: {e}" print(json.dumps(result, indent=2)) return 1 diff --git a/isvctl/configs/providers/aws/scripts/block-storage/snapshot_lifecycle.py b/isvctl/configs/providers/aws/scripts/block-storage/snapshot_lifecycle.py index 2450495f..51740111 100644 --- a/isvctl/configs/providers/aws/scripts/block-storage/snapshot_lifecycle.py +++ b/isvctl/configs/providers/aws/scripts/block-storage/snapshot_lifecycle.py @@ -48,7 +48,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # providers/aws/scripts/ (for common.*) import boto3 -from botocore.exceptions import ClientError, NoCredentialsError +from botocore.exceptions import BotoCoreError, ClientError from common import ebs from common.ssh_utils import ssh_run, wait_for_ssh @@ -107,10 +107,12 @@ def main() -> int: result["snapshot_id"] = snapshot_id ebs.wait_for_snapshot_completed(ec2, snapshot_id) operations["create_snapshot"]["passed"] = True - except (ClientError, NoCredentialsError) as e: + except (ClientError, BotoCoreError) as e: _fail(operations["create_snapshot"], str(e)) result["error"] = f"CreateSnapshot failed: {e}" - return _emit(result) + # snapshot_id may be set if creation succeeded but the wait failed - + # pass it so a partially-created snapshot is still cleaned up. + return _emit(result, ec2, restored_volume_id, snapshot_id) try: restored_volume_id = ebs.create_volume_from_snapshot(ec2, snapshot_id, availability_zone) @@ -119,7 +121,7 @@ def main() -> int: ebs.attach_volume(ec2, restored_volume_id, args.instance_id, args.restore_device) ebs.wait_for_volume_in_use(ec2, restored_volume_id) operations["restore_volume"]["passed"] = True - except ClientError as e: + except (ClientError, BotoCoreError) as e: _fail(operations["restore_volume"], str(e)) result["error"] = f"Restore failed: {e}" return _emit(result, ec2, restored_volume_id, snapshot_id) @@ -149,7 +151,7 @@ def main() -> int: result["success"] = all(op["passed"] for op in operations.values()) return _emit(result, ec2, restored_volume_id, snapshot_id) - except (ClientError, NoCredentialsError) as e: + except (ClientError, BotoCoreError) as e: result["error"] = str(e) return _emit(result, ec2, restored_volume_id, snapshot_id) diff --git a/isvctl/configs/providers/aws/scripts/block-storage/volume_persistence.py b/isvctl/configs/providers/aws/scripts/block-storage/volume_persistence.py index 5bf0812d..3d8be1f3 100644 --- a/isvctl/configs/providers/aws/scripts/block-storage/volume_persistence.py +++ b/isvctl/configs/providers/aws/scripts/block-storage/volume_persistence.py @@ -46,7 +46,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # providers/aws/scripts/ (for common.*) import boto3 -from botocore.exceptions import ClientError, NoCredentialsError +from botocore.exceptions import BotoCoreError, ClientError from common import ebs from common.ec2 import wait_for_public_ip from common.ssh_utils import wait_for_ssh @@ -112,7 +112,7 @@ def main() -> int: print(json.dumps(result, indent=2)) return 1 operations["start"]["passed"] = True - except (ClientError, NoCredentialsError) as e: + except (ClientError, BotoCoreError) as e: result["error"] = f"Restart failed: {e}" print(json.dumps(result, indent=2)) return 1 diff --git a/isvctl/configs/providers/aws/scripts/block-storage/volume_resize.py b/isvctl/configs/providers/aws/scripts/block-storage/volume_resize.py index 25ba166e..53906258 100644 --- a/isvctl/configs/providers/aws/scripts/block-storage/volume_resize.py +++ b/isvctl/configs/providers/aws/scripts/block-storage/volume_resize.py @@ -50,7 +50,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # providers/aws/scripts/ (for common.*) import boto3 -from botocore.exceptions import ClientError, NoCredentialsError +from botocore.exceptions import BotoCoreError, ClientError from common import ebs from common.ssh_utils import ssh_run, wait_for_ssh @@ -124,12 +124,12 @@ def main() -> int: ebs.modify_volume_size(ec2, args.volume_id, new_size) ebs.wait_for_modification_complete(ec2, args.volume_id) operations["modify_volume"]["passed"] = True - except (ClientError, NoCredentialsError, RuntimeError) as e: + except (ClientError, BotoCoreError, RuntimeError) as e: _fail(operations["modify_volume"], str(e)) result["error"] = f"ModifyVolume failed: {e}" print(json.dumps(result, indent=2)) return 1 - except (ClientError, NoCredentialsError) as e: + except (ClientError, BotoCoreError) as e: result["error"] = str(e) print(json.dumps(result, indent=2)) return 1 diff --git a/isvctl/configs/providers/aws/scripts/common/ebs.py b/isvctl/configs/providers/aws/scripts/common/ebs.py index 4c5c34be..9e03277b 100644 --- a/isvctl/configs/providers/aws/scripts/common/ebs.py +++ b/isvctl/configs/providers/aws/scripts/common/ebs.py @@ -37,7 +37,7 @@ import time from typing import Any -from botocore.exceptions import ClientError +from botocore.exceptions import BotoCoreError, ClientError # Ownership tag so cleanup and audits can tell suite-created volumes apart # from anything the account already had. @@ -249,6 +249,10 @@ def detach_and_delete_volume(ec2: Any, volume_id: str) -> str | None: if code in _NOT_FOUND_CODES: return None return str(e) + except BotoCoreError as e: + # Waiter timeouts / transport errors must not propagate out of a + # best-effort cleanup path (callers run this in finally blocks). + return str(e) def delete_snapshot_best_effort(ec2: Any, snapshot_id: str) -> str | None: @@ -261,6 +265,8 @@ def delete_snapshot_best_effort(ec2: Any, snapshot_id: str) -> str | None: if code in _NOT_FOUND_CODES: return None return str(e) + except BotoCoreError as e: + return str(e) def wait_for_attachment_device(host: str, user: str, key_file: str, volume_id: str, *, attempts: int = 30) -> bool: From accaea7ac7f28bcc083bf20a6a58c5099e0340b9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 12 Jun 2026 19:18:50 +0000 Subject: [PATCH 06/10] refactor(block-storage): fold validations into vm suite instead of a new suite Move the DATASVC-XX-02/03/04 block-storage validations out of the standalone suites/block-storage.yaml and into suites/vm.yaml, so block storage is validated under the existing VM contract rather than adding another suite file. - vm.yaml: add fixture_volume / snapshot_lifecycle / volume_resize / volume_persistence / volume_teardown_checks. These activate only when a provider config supplies the matching volume steps; a standard VM run skips them automatically. Renamed the volume teardown check to volume_teardown_checks to avoid colliding with the VM teardown_checks, and dropped the redundant instance_launched (covered by setup_checks). - providers/aws|my-isv/config/block-storage.yaml: import suites/vm.yaml, rename the command group to vm (must match the inherited platform), and exclude the ssh-labelled VM host checks (cloud-init/GPU/OS/NIM are not block-storage concerns). - delete suites/block-storage.yaml. - docs: drop block-storage from the suites list and point the domain's contract at vm.yaml. Signed-off-by: Cursor Agent Co-authored-by: Alexandre Begnoche --- .../providers/aws/config/block-storage.yaml | 23 +++- .../my-isv/config/block-storage.yaml | 22 +++- .../providers/my-isv/scripts/README.md | 2 +- isvctl/configs/suites/README.md | 10 +- isvctl/configs/suites/block-storage.yaml | 119 ------------------ isvctl/configs/suites/vm.yaml | 56 +++++++++ 6 files changed, 98 insertions(+), 134 deletions(-) delete mode 100644 isvctl/configs/suites/block-storage.yaml diff --git a/isvctl/configs/providers/aws/config/block-storage.yaml b/isvctl/configs/providers/aws/config/block-storage.yaml index ba8552a4..b159b30e 100644 --- a/isvctl/configs/providers/aws/config/block-storage.yaml +++ b/isvctl/configs/providers/aws/config/block-storage.yaml @@ -15,10 +15,14 @@ # AWS Block Storage (EBS) Validation Configuration # -# Imports validations from the provider-agnostic block-storage canonical -# config. This file only provides AWS-specific commands (boto3 + SSH) and -# settings. The launch_instance / teardown steps reuse the VM reference -# scripts; the volume steps live in scripts/block-storage/. +# Block storage is validated as part of the VM contract, so this config +# imports suites/vm.yaml and supplies only the volume steps. The VM +# validations whose steps are omitted here are skipped automatically; the +# block-storage validations (fixture_volume / snapshot_lifecycle / +# volume_resize / volume_persistence) activate because the matching steps +# below are present. This file only provides AWS-specific commands +# (boto3 + SSH) and settings. The launch_instance / teardown steps reuse +# the VM reference scripts; the volume steps live in scripts/block-storage/. # # Steps: # SETUP @@ -45,12 +49,12 @@ # ec2:CreateSnapshot, ec2:DeleteSnapshot, ec2:DescribeSnapshots import: - - ../../../suites/block-storage.yaml + - ../../../suites/vm.yaml version: "1.0" commands: - block_storage: + vm: phases: ["setup", "test", "teardown"] steps: # AWS-specific: launch the fixture instance (reuses the VM script) @@ -170,3 +174,10 @@ tests: instance_type: "m6i.large" volume_size_gib: "10" teardown_flag: "{{(env.BLOCK_STORAGE_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" + + # Block storage reuses the VM contract (suites/vm.yaml) for its fixture but + # does not validate the VM host itself. The ssh-labelled VM host checks + # (cloud-init / GPU / OS / driver / NIM, etc.) are excluded; the VM checks + # whose steps this config omits are skipped automatically. + exclude: + labels: ["ssh"] diff --git a/isvctl/configs/providers/my-isv/config/block-storage.yaml b/isvctl/configs/providers/my-isv/config/block-storage.yaml index 66748479..d17cf117 100644 --- a/isvctl/configs/providers/my-isv/config/block-storage.yaml +++ b/isvctl/configs/providers/my-isv/config/block-storage.yaml @@ -15,10 +15,13 @@ # my-isv Block Storage Validation Configuration - Living Example # -# Imports validations from the provider-agnostic block-storage template and -# wires the commands to the generic template scripts. Those scripts ship with -# dummy-success values so this example runs end-to-end without any real cloud -# infrastructure (ISVCTL_DEMO_MODE=1, used by `make demo-test`). +# Block storage is validated as part of the VM contract, so this config +# imports suites/vm.yaml and supplies only the volume steps. VM validations +# whose steps are omitted here are skipped automatically; the block-storage +# validations activate because the matching steps below are present. The +# commands wire to the generic template scripts, which ship with dummy-success +# values so this example runs end-to-end without any real cloud infrastructure +# (ISVCTL_DEMO_MODE=1, used by `make demo-test`). # # The launch_instance / teardown steps reuse the VM scaffold scripts; the # volume steps live in scripts/block-storage/. @@ -39,12 +42,12 @@ # uv run isvctl test run -f isvctl/configs/providers/my-isv/config/block-storage.yaml import: - - ../../../suites/block-storage.yaml + - ../../../suites/vm.yaml version: "1.0" commands: - block_storage: + vm: phases: ["setup", "test", "teardown"] steps: - name: launch_instance @@ -155,3 +158,10 @@ tests: instance_type: "my-isv.standard.1x" volume_size_gib: "10" teardown_flag: "{{(env.BLOCK_STORAGE_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" + + # Block storage reuses the VM contract (suites/vm.yaml) for its fixture but + # does not validate the VM host itself. The ssh-labelled VM host checks + # (cloud-init / GPU / OS / driver / NIM, etc.) are excluded; the VM checks + # whose steps this config omits are skipped automatically. + exclude: + labels: ["ssh"] diff --git a/isvctl/configs/providers/my-isv/scripts/README.md b/isvctl/configs/providers/my-isv/scripts/README.md index 4bbae0eb..18fc83fa 100644 --- a/isvctl/configs/providers/my-isv/scripts/README.md +++ b/isvctl/configs/providers/my-isv/scripts/README.md @@ -31,7 +31,7 @@ template, then fill in the TODOs. | `control-plane/` | 10 | [`suites/control-plane.yaml`](../../../suites/control-plane.yaml) | [`config/control-plane.yaml`](../config/control-plane.yaml) | [`providers/aws/scripts/control-plane/`](../../aws/scripts/control-plane/) | | `vm/` | 9 | [`suites/vm.yaml`](../../../suites/vm.yaml) | [`config/vm.yaml`](../config/vm.yaml) | [`providers/aws/scripts/vm/`](../../aws/scripts/vm/) | | `bare_metal/` | 12 | [`suites/bare_metal.yaml`](../../../suites/bare_metal.yaml) | [`config/bare_metal.yaml`](../config/bare_metal.yaml) | [`providers/aws/scripts/bare_metal/`](../../aws/scripts/bare_metal/) | -| `block-storage/` | 5 | [`suites/block-storage.yaml`](../../../suites/block-storage.yaml) | [`config/block-storage.yaml`](../config/block-storage.yaml) | [`providers/aws/scripts/block-storage/`](../../aws/scripts/block-storage/) | +| `block-storage/` | 5 | [`suites/vm.yaml`](../../../suites/vm.yaml) | [`config/block-storage.yaml`](../config/block-storage.yaml) | [`providers/aws/scripts/block-storage/`](../../aws/scripts/block-storage/) | | `network/` | 18 | [`suites/network.yaml`](../../../suites/network.yaml) | [`config/network.yaml`](../config/network.yaml) | [`providers/aws/scripts/network/`](../../aws/scripts/network/) | | `observability/` | 1 | [`suites/observability.yaml`](../../../suites/observability.yaml) | [`config/observability.yaml`](../config/observability.yaml) | [`providers/aws/scripts/observability/`](../../aws/scripts/observability/) | | `image-registry/` | 7 | [`suites/image-registry.yaml`](../../../suites/image-registry.yaml) | [`config/image-registry.yaml`](../config/image-registry.yaml) | [`providers/aws/scripts/image-registry/`](../../aws/scripts/image-registry/) | diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index ed4718e5..c6535f4c 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -14,7 +14,6 @@ Suites: [`network`](network.yaml), [`vm`](vm.yaml), [`bare_metal`](bare_metal.yaml), -[`block-storage`](block-storage.yaml), [`observability`](observability.yaml), [`k8s`](k8s.yaml), [`slurm`](slurm.yaml), @@ -122,7 +121,14 @@ For the domain / script-count / AWS-reference overview see the | `query_sanitization` | test | `providers/nico/scripts/sanitization/query_sanitization.py` | `machines_checked`, `machines[].{available,in_use,has_gpu,served_tenant,sanitized,stale_tenant_binding,vendor,product_name,bios_version,transitions}` | | `query_attestation` | test | `providers/nico/scripts/attestation/query_attestation.py` | `machines_checked`, `machines[].{attestation_supported,nonce_verified,attestation_signature_valid,secure_boot_enabled,boot_measurements_attested,measured_boot_state}` | -### Block Storage (`block-storage.yaml`) +### Block Storage (part of `vm.yaml`) + +Block storage (DATASVC-XX-02/03/04) is validated as part of the [VM](vm.yaml) contract +rather than a standalone suite. The block-storage validations +(`fixture_volume` / `snapshot_lifecycle` / `volume_resize` / `volume_persistence` / +`volume_teardown_checks`) live in `vm.yaml` and activate only when a provider config +supplies the matching volume steps - see `providers/

/config/block-storage.yaml`. A +standard VM run that omits those steps skips these checks automatically. A shared fixture (`launch_instance` + `create_volume`) provisions one instance with a single attached, formatted, mounted, and seeded block volume. The three test-phase diff --git a/isvctl/configs/suites/block-storage.yaml b/isvctl/configs/suites/block-storage.yaml deleted file mode 100644 index 304a0b7a..00000000 --- a/isvctl/configs/suites/block-storage.yaml +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Block Storage Validation - Canonical Contract -# -# Validations-only contract for persistent block storage (EBS-style -# volumes). Provider configs import this file and supply their own -# commands (steps + scripts). -# -# This file defines WHAT to validate, not HOW to run it. Each validation -# only inspects JSON output fields. As long as your scripts output the -# right fields, the validations pass regardless of which cloud you use. -# -# Coverage notes: -# - A shared fixture launches one instance and creates + attaches a single -# block volume (formatted, mounted, seeded with a sentinel file). All -# three test-phase checks reuse that fixture: -# - snapshot_lifecycle - DATASVC-XX-02 (issue #321): snapshot the volume, -# restore it to a new volume, attach + mount the restore, and verify the -# sentinel data survived the round-trip. -# - volume_resize - DATASVC-XX-03 (issue #322): grow the volume -# (ModifyVolume), then grow the in-guest partition + filesystem and -# confirm the larger size is visible to the guest. -# - volume_persistence - DATASVC-XX-04 (issue #323): stop + start the -# instance and confirm the volume stays attached with its data intact. -# -# Quick start: -# 1. Copy providers/my-isv/config/block-storage.yaml to providers//config/block-storage.yaml -# 2. Replace stub paths with your platform scripts -# 3. Run: uv run isvctl test run -f isvctl/configs/providers//config/block-storage.yaml -# -# See also: -# - my-isv living example: providers/my-isv/config/block-storage.yaml -# - AWS reference: providers/aws/config/block-storage.yaml + providers/aws/scripts/block-storage/ -# - Per-step field breakdown: ../suites/README.md - -version: "1.0" - -tests: - platform: block_storage - cluster_name: "block-storage-validation" - description: "Block storage: snapshots, resizing, and persistence across restarts" - - settings: - # Provider-specific - override in your provider config - region: "" - instance_type: "" - volume_size_gib: "10" - teardown_flag: "{{(env.BLOCK_STORAGE_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" - - # ─── Validations ───────────────────────────────────────────────── - # These are PROVIDER-AGNOSTIC. They only check JSON field names/values. - validations: - # ─── Fixture: instance + attached, formatted, seeded volume ────── - instance_launched: - step: launch_instance - checks: - InstanceStateCheck: - expected_state: "running" - - fixture_volume: - step: create_volume - checks: - StepSuccessCheck: {} - FieldExistsCheck: - fields: ["volume_id", "mount_point", "operations"] - CrudOperationsCheck: - operations: ["create", "attach", "format", "mount", "write_sentinel"] - - # ─── DATASVC-XX-02: volume snapshots (issue #321) ──────────────── - snapshot_lifecycle: - step: snapshot_lifecycle - checks: - StepSuccessCheck: {} - FieldExistsCheck: - fields: ["volume_id", "snapshot_id", "operations"] - CrudOperationsCheck: - operations: ["create_snapshot", "restore_volume", "verify_data"] - - # ─── DATASVC-XX-03: volume resizing (issue #322) ───────────────── - volume_resize: - step: volume_resize - checks: - StepSuccessCheck: {} - FieldExistsCheck: - fields: ["volume_id", "operations"] - CrudOperationsCheck: - operations: ["modify_volume", "grow_partition", "resize_filesystem", "verify_size"] - - # ─── DATASVC-XX-04: persistence across restarts (issue #323) ───── - volume_persistence: - step: volume_persistence - checks: - StepSuccessCheck: {} - FieldExistsCheck: - fields: ["volume_id", "operations"] - CrudOperationsCheck: - operations: ["stop", "start", "verify_attached", "verify_data"] - - # ─── Teardown ──────────────────────────────────────────────────── - teardown_checks: - step: teardown_volume - checks: - StepSuccessCheck: {} - - exclude: - labels: [] diff --git a/isvctl/configs/suites/vm.yaml b/isvctl/configs/suites/vm.yaml index af0f2efb..dfba68ce 100644 --- a/isvctl/configs/suites/vm.yaml +++ b/isvctl/configs/suites/vm.yaml @@ -306,5 +306,61 @@ tests: test_id: "N/A" labels: ["vm"] + # ─── Block storage validations (DATASVC-XX-02/03/04) ───────────── + # Optional persistent block-volume coverage. These only activate when + # the provider config supplies the matching volume steps (see + # providers/

/config/block-storage.yaml). A standard VM run that omits + # those steps skips these checks automatically. + # + # A shared fixture launches one instance (setup_checks above) and creates + # + attaches a single block volume (create_volume), formatted, mounted, and + # seeded with a sentinel file. The three test-phase checks reuse it: + # - snapshot_lifecycle - DATASVC-XX-02 (#321): snapshot -> restore -> + # re-attach + mount the restore, verify the sentinel survived. + # - volume_resize - DATASVC-XX-03 (#322): grow the volume, then the + # in-guest partition + filesystem, and confirm the guest sees it. + # - volume_persistence - DATASVC-XX-04 (#323): stop + start the instance + # and confirm the volume stays attached with its data intact. + fixture_volume: + step: create_volume + checks: + StepSuccessCheck: {} + FieldExistsCheck: + fields: ["volume_id", "mount_point", "operations"] + CrudOperationsCheck: + operations: ["create", "attach", "format", "mount", "write_sentinel"] + + snapshot_lifecycle: + step: snapshot_lifecycle + checks: + StepSuccessCheck: {} + FieldExistsCheck: + fields: ["volume_id", "snapshot_id", "operations"] + CrudOperationsCheck: + operations: ["create_snapshot", "restore_volume", "verify_data"] + + volume_resize: + step: volume_resize + checks: + StepSuccessCheck: {} + FieldExistsCheck: + fields: ["volume_id", "operations"] + CrudOperationsCheck: + operations: ["modify_volume", "grow_partition", "resize_filesystem", "verify_size"] + + volume_persistence: + step: volume_persistence + checks: + StepSuccessCheck: {} + FieldExistsCheck: + fields: ["volume_id", "operations"] + CrudOperationsCheck: + operations: ["stop", "start", "verify_attached", "verify_data"] + + volume_teardown_checks: + step: teardown_volume + checks: + StepSuccessCheck: {} + exclude: labels: [] From 8dbbcc454ae94d748fa1c82c69b0129a8925ca70 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 12 Jun 2026 19:48:08 +0000 Subject: [PATCH 07/10] refactor(storage): host block-storage in a dedicated storage suite Replace the earlier approach of folding block-storage validations into suites/vm.yaml. Instead, add a dedicated suites/storage.yaml umbrella contract (platform: storage) that today covers persistent block storage (DATASVC-XX-02/03/04) and is the future home for object/file storage checks - avoiding both suite proliferation and the skip-noise of sharing the VM contract. - suites/storage.yaml: new umbrella contract (was the block-storage suite). - suites/vm.yaml: reverted to its original VM-only validations. - providers/aws|my-isv/config/block-storage.yaml: import suites/storage.yaml and use the 'storage' command group (matches the suite platform). - docs: list the 'storage' suite and point the block-storage domain's contract at storage.yaml. The block-storage provider domain (config/block-storage.yaml + scripts/block-storage/) stays as the concrete block-volume implementation of the storage contract. Signed-off-by: Cursor Agent Co-authored-by: Alexandre Begnoche --- .../providers/aws/config/block-storage.yaml | 24 +--- .../my-isv/config/block-storage.yaml | 23 +--- .../providers/my-isv/scripts/README.md | 2 +- isvctl/configs/suites/README.md | 18 +-- isvctl/configs/suites/storage.yaml | 122 ++++++++++++++++++ isvctl/configs/suites/vm.yaml | 56 -------- 6 files changed, 144 insertions(+), 101 deletions(-) create mode 100644 isvctl/configs/suites/storage.yaml diff --git a/isvctl/configs/providers/aws/config/block-storage.yaml b/isvctl/configs/providers/aws/config/block-storage.yaml index b159b30e..892ac180 100644 --- a/isvctl/configs/providers/aws/config/block-storage.yaml +++ b/isvctl/configs/providers/aws/config/block-storage.yaml @@ -15,14 +15,11 @@ # AWS Block Storage (EBS) Validation Configuration # -# Block storage is validated as part of the VM contract, so this config -# imports suites/vm.yaml and supplies only the volume steps. The VM -# validations whose steps are omitted here are skipped automatically; the -# block-storage validations (fixture_volume / snapshot_lifecycle / -# volume_resize / volume_persistence) activate because the matching steps -# below are present. This file only provides AWS-specific commands -# (boto3 + SSH) and settings. The launch_instance / teardown steps reuse -# the VM reference scripts; the volume steps live in scripts/block-storage/. +# Block storage is the block-volume implementation of the storage suite, so +# this config imports the provider-agnostic suites/storage.yaml and supplies +# AWS-specific commands (boto3 + SSH) and settings. The launch_instance / +# teardown steps reuse the VM reference scripts; the volume steps live in +# scripts/block-storage/. # # Steps: # SETUP @@ -49,12 +46,12 @@ # ec2:CreateSnapshot, ec2:DeleteSnapshot, ec2:DescribeSnapshots import: - - ../../../suites/vm.yaml + - ../../../suites/storage.yaml version: "1.0" commands: - vm: + storage: phases: ["setup", "test", "teardown"] steps: # AWS-specific: launch the fixture instance (reuses the VM script) @@ -174,10 +171,3 @@ tests: instance_type: "m6i.large" volume_size_gib: "10" teardown_flag: "{{(env.BLOCK_STORAGE_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" - - # Block storage reuses the VM contract (suites/vm.yaml) for its fixture but - # does not validate the VM host itself. The ssh-labelled VM host checks - # (cloud-init / GPU / OS / driver / NIM, etc.) are excluded; the VM checks - # whose steps this config omits are skipped automatically. - exclude: - labels: ["ssh"] diff --git a/isvctl/configs/providers/my-isv/config/block-storage.yaml b/isvctl/configs/providers/my-isv/config/block-storage.yaml index d17cf117..28ab6907 100644 --- a/isvctl/configs/providers/my-isv/config/block-storage.yaml +++ b/isvctl/configs/providers/my-isv/config/block-storage.yaml @@ -15,13 +15,11 @@ # my-isv Block Storage Validation Configuration - Living Example # -# Block storage is validated as part of the VM contract, so this config -# imports suites/vm.yaml and supplies only the volume steps. VM validations -# whose steps are omitted here are skipped automatically; the block-storage -# validations activate because the matching steps below are present. The -# commands wire to the generic template scripts, which ship with dummy-success -# values so this example runs end-to-end without any real cloud infrastructure -# (ISVCTL_DEMO_MODE=1, used by `make demo-test`). +# Block storage is the block-volume implementation of the storage suite, so +# this config imports the provider-agnostic suites/storage.yaml and wires the +# commands to the generic template scripts. Those scripts ship with +# dummy-success values so this example runs end-to-end without any real cloud +# infrastructure (ISVCTL_DEMO_MODE=1, used by `make demo-test`). # # The launch_instance / teardown steps reuse the VM scaffold scripts; the # volume steps live in scripts/block-storage/. @@ -42,12 +40,12 @@ # uv run isvctl test run -f isvctl/configs/providers/my-isv/config/block-storage.yaml import: - - ../../../suites/vm.yaml + - ../../../suites/storage.yaml version: "1.0" commands: - vm: + storage: phases: ["setup", "test", "teardown"] steps: - name: launch_instance @@ -158,10 +156,3 @@ tests: instance_type: "my-isv.standard.1x" volume_size_gib: "10" teardown_flag: "{{(env.BLOCK_STORAGE_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" - - # Block storage reuses the VM contract (suites/vm.yaml) for its fixture but - # does not validate the VM host itself. The ssh-labelled VM host checks - # (cloud-init / GPU / OS / driver / NIM, etc.) are excluded; the VM checks - # whose steps this config omits are skipped automatically. - exclude: - labels: ["ssh"] diff --git a/isvctl/configs/providers/my-isv/scripts/README.md b/isvctl/configs/providers/my-isv/scripts/README.md index 18fc83fa..f614bca6 100644 --- a/isvctl/configs/providers/my-isv/scripts/README.md +++ b/isvctl/configs/providers/my-isv/scripts/README.md @@ -31,7 +31,7 @@ template, then fill in the TODOs. | `control-plane/` | 10 | [`suites/control-plane.yaml`](../../../suites/control-plane.yaml) | [`config/control-plane.yaml`](../config/control-plane.yaml) | [`providers/aws/scripts/control-plane/`](../../aws/scripts/control-plane/) | | `vm/` | 9 | [`suites/vm.yaml`](../../../suites/vm.yaml) | [`config/vm.yaml`](../config/vm.yaml) | [`providers/aws/scripts/vm/`](../../aws/scripts/vm/) | | `bare_metal/` | 12 | [`suites/bare_metal.yaml`](../../../suites/bare_metal.yaml) | [`config/bare_metal.yaml`](../config/bare_metal.yaml) | [`providers/aws/scripts/bare_metal/`](../../aws/scripts/bare_metal/) | -| `block-storage/` | 5 | [`suites/vm.yaml`](../../../suites/vm.yaml) | [`config/block-storage.yaml`](../config/block-storage.yaml) | [`providers/aws/scripts/block-storage/`](../../aws/scripts/block-storage/) | +| `block-storage/` | 5 | [`suites/storage.yaml`](../../../suites/storage.yaml) | [`config/block-storage.yaml`](../config/block-storage.yaml) | [`providers/aws/scripts/block-storage/`](../../aws/scripts/block-storage/) | | `network/` | 18 | [`suites/network.yaml`](../../../suites/network.yaml) | [`config/network.yaml`](../config/network.yaml) | [`providers/aws/scripts/network/`](../../aws/scripts/network/) | | `observability/` | 1 | [`suites/observability.yaml`](../../../suites/observability.yaml) | [`config/observability.yaml`](../config/observability.yaml) | [`providers/aws/scripts/observability/`](../../aws/scripts/observability/) | | `image-registry/` | 7 | [`suites/image-registry.yaml`](../../../suites/image-registry.yaml) | [`config/image-registry.yaml`](../config/image-registry.yaml) | [`providers/aws/scripts/image-registry/`](../../aws/scripts/image-registry/) | diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index c6535f4c..08c2680f 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -14,6 +14,7 @@ Suites: [`network`](network.yaml), [`vm`](vm.yaml), [`bare_metal`](bare_metal.yaml), +[`storage`](storage.yaml), [`observability`](observability.yaml), [`k8s`](k8s.yaml), [`slurm`](slurm.yaml), @@ -121,18 +122,13 @@ For the domain / script-count / AWS-reference overview see the | `query_sanitization` | test | `providers/nico/scripts/sanitization/query_sanitization.py` | `machines_checked`, `machines[].{available,in_use,has_gpu,served_tenant,sanitized,stale_tenant_binding,vendor,product_name,bios_version,transitions}` | | `query_attestation` | test | `providers/nico/scripts/attestation/query_attestation.py` | `machines_checked`, `machines[].{attestation_supported,nonce_verified,attestation_signature_valid,secure_boot_enabled,boot_measurements_attested,measured_boot_state}` | -### Block Storage (part of `vm.yaml`) +### Storage (`storage.yaml`) -Block storage (DATASVC-XX-02/03/04) is validated as part of the [VM](vm.yaml) contract -rather than a standalone suite. The block-storage validations -(`fixture_volume` / `snapshot_lifecycle` / `volume_resize` / `volume_persistence` / -`volume_teardown_checks`) live in `vm.yaml` and activate only when a provider config -supplies the matching volume steps - see `providers/

/config/block-storage.yaml`. A -standard VM run that omits those steps skips these checks automatically. - -A shared fixture (`launch_instance` + `create_volume`) provisions one instance with a -single attached, formatted, mounted, and seeded block volume. The three test-phase -steps (DATASVC-XX-02/03/04) all reuse that fixture. +Umbrella suite for the storage capability area. Today it covers persistent block +storage (DATASVC-XX-02/03/04); future object/file storage checks land here too rather +than spawning new suites. A shared fixture (`launch_instance` + `create_volume`) +provisions one instance with a single attached, formatted, mounted, and seeded block +volume. The three test-phase steps all reuse that fixture. | Step | Phase | Script | Key JSON Fields | |------|-------|--------|-----------------| diff --git a/isvctl/configs/suites/storage.yaml b/isvctl/configs/suites/storage.yaml new file mode 100644 index 00000000..ce92d14e --- /dev/null +++ b/isvctl/configs/suites/storage.yaml @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Storage Validation - Canonical Contract +# +# Validations-only contract for the storage capability area. Provider +# configs import this file and supply their own commands (steps + scripts). +# +# This file defines WHAT to validate, not HOW to run it. Each validation +# only inspects JSON output fields. As long as your scripts output the +# right fields, the validations pass regardless of which cloud you use. +# +# Scope: today this suite covers persistent block storage (EBS-style +# volumes); it is the umbrella home for future storage checks (object, +# file, ...) so they don't each spawn a new suite file. +# +# Coverage notes: +# - A shared fixture launches one instance and creates + attaches a single +# block volume (formatted, mounted, seeded with a sentinel file). All +# three test-phase checks reuse that fixture: +# - snapshot_lifecycle - DATASVC-XX-02 (issue #321): snapshot the volume, +# restore it to a new volume, attach + mount the restore, and verify the +# sentinel data survived the round-trip. +# - volume_resize - DATASVC-XX-03 (issue #322): grow the volume +# (ModifyVolume), then grow the in-guest partition + filesystem and +# confirm the larger size is visible to the guest. +# - volume_persistence - DATASVC-XX-04 (issue #323): stop + start the +# instance and confirm the volume stays attached with its data intact. +# +# Quick start: +# 1. Copy providers/my-isv/config/block-storage.yaml to providers//config/block-storage.yaml +# 2. Replace stub paths with your platform scripts +# 3. Run: uv run isvctl test run -f isvctl/configs/providers//config/block-storage.yaml +# +# See also: +# - my-isv living example: providers/my-isv/config/block-storage.yaml +# - AWS reference: providers/aws/config/block-storage.yaml + providers/aws/scripts/block-storage/ +# - Per-step field breakdown: ../suites/README.md + +version: "1.0" + +tests: + platform: storage + cluster_name: "storage-validation" + description: "Storage: block-volume snapshots, resizing, and persistence across restarts" + + settings: + # Provider-specific - override in your provider config + region: "" + instance_type: "" + volume_size_gib: "10" + teardown_flag: "{{(env.BLOCK_STORAGE_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" + + # ─── Validations ───────────────────────────────────────────────── + # These are PROVIDER-AGNOSTIC. They only check JSON field names/values. + validations: + # ─── Fixture: instance + attached, formatted, seeded volume ────── + instance_launched: + step: launch_instance + checks: + InstanceStateCheck: + expected_state: "running" + + fixture_volume: + step: create_volume + checks: + StepSuccessCheck: {} + FieldExistsCheck: + fields: ["volume_id", "mount_point", "operations"] + CrudOperationsCheck: + operations: ["create", "attach", "format", "mount", "write_sentinel"] + + # ─── DATASVC-XX-02: volume snapshots (issue #321) ──────────────── + snapshot_lifecycle: + step: snapshot_lifecycle + checks: + StepSuccessCheck: {} + FieldExistsCheck: + fields: ["volume_id", "snapshot_id", "operations"] + CrudOperationsCheck: + operations: ["create_snapshot", "restore_volume", "verify_data"] + + # ─── DATASVC-XX-03: volume resizing (issue #322) ───────────────── + volume_resize: + step: volume_resize + checks: + StepSuccessCheck: {} + FieldExistsCheck: + fields: ["volume_id", "operations"] + CrudOperationsCheck: + operations: ["modify_volume", "grow_partition", "resize_filesystem", "verify_size"] + + # ─── DATASVC-XX-04: persistence across restarts (issue #323) ───── + volume_persistence: + step: volume_persistence + checks: + StepSuccessCheck: {} + FieldExistsCheck: + fields: ["volume_id", "operations"] + CrudOperationsCheck: + operations: ["stop", "start", "verify_attached", "verify_data"] + + # ─── Teardown ──────────────────────────────────────────────────── + teardown_checks: + step: teardown_volume + checks: + StepSuccessCheck: {} + + exclude: + labels: [] diff --git a/isvctl/configs/suites/vm.yaml b/isvctl/configs/suites/vm.yaml index dfba68ce..af0f2efb 100644 --- a/isvctl/configs/suites/vm.yaml +++ b/isvctl/configs/suites/vm.yaml @@ -306,61 +306,5 @@ tests: test_id: "N/A" labels: ["vm"] - # ─── Block storage validations (DATASVC-XX-02/03/04) ───────────── - # Optional persistent block-volume coverage. These only activate when - # the provider config supplies the matching volume steps (see - # providers/

/config/block-storage.yaml). A standard VM run that omits - # those steps skips these checks automatically. - # - # A shared fixture launches one instance (setup_checks above) and creates - # + attaches a single block volume (create_volume), formatted, mounted, and - # seeded with a sentinel file. The three test-phase checks reuse it: - # - snapshot_lifecycle - DATASVC-XX-02 (#321): snapshot -> restore -> - # re-attach + mount the restore, verify the sentinel survived. - # - volume_resize - DATASVC-XX-03 (#322): grow the volume, then the - # in-guest partition + filesystem, and confirm the guest sees it. - # - volume_persistence - DATASVC-XX-04 (#323): stop + start the instance - # and confirm the volume stays attached with its data intact. - fixture_volume: - step: create_volume - checks: - StepSuccessCheck: {} - FieldExistsCheck: - fields: ["volume_id", "mount_point", "operations"] - CrudOperationsCheck: - operations: ["create", "attach", "format", "mount", "write_sentinel"] - - snapshot_lifecycle: - step: snapshot_lifecycle - checks: - StepSuccessCheck: {} - FieldExistsCheck: - fields: ["volume_id", "snapshot_id", "operations"] - CrudOperationsCheck: - operations: ["create_snapshot", "restore_volume", "verify_data"] - - volume_resize: - step: volume_resize - checks: - StepSuccessCheck: {} - FieldExistsCheck: - fields: ["volume_id", "operations"] - CrudOperationsCheck: - operations: ["modify_volume", "grow_partition", "resize_filesystem", "verify_size"] - - volume_persistence: - step: volume_persistence - checks: - StepSuccessCheck: {} - FieldExistsCheck: - fields: ["volume_id", "operations"] - CrudOperationsCheck: - operations: ["stop", "start", "verify_attached", "verify_data"] - - volume_teardown_checks: - step: teardown_volume - checks: - StepSuccessCheck: {} - exclude: labels: [] From 25bca23dec4acb048fab2b264e53da20d6a448e3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 12 Jun 2026 19:54:42 +0000 Subject: [PATCH 08/10] refactor(storage): rename block-storage provider paths to storage Complete the storage-suite rename by aligning the runnable provider domain with the suite name. - providers/aws|my-isv/config/storage.yaml replace config/block-storage.yaml. - providers/aws|my-isv/scripts/storage/ replace scripts/block-storage/. - Makefile demo domain is now storage, so the focused smoke test is make demo-storage. - Tests and docs now refer to storage paths and the AWS unit test file is renamed to test_aws_storage_scripts.py. Signed-off-by: Cursor Agent Co-authored-by: Alexandre Begnoche --- Makefile | 2 +- .../{block-storage.yaml => storage.yaml} | 20 ++++++------- .../providers/aws/scripts/common/ebs.py | 4 +-- .../create_volume.py | 10 +++---- .../snapshot_lifecycle.py | 4 +-- .../teardown_volume.py | 8 +++--- .../volume_persistence.py | 4 +-- .../volume_resize.py | 12 +++++--- .../{block-storage.yaml => storage.yaml} | 20 ++++++------- .../providers/my-isv/scripts/README.md | 2 +- .../create_volume.py | 8 +++--- .../snapshot_lifecycle.py | 6 ++-- .../teardown_volume.py | 10 +++---- .../volume_persistence.py | 6 ++-- .../volume_resize.py | 6 ++-- isvctl/configs/suites/README.md | 10 +++---- isvctl/configs/suites/storage.yaml | 8 +++--- ...scripts.py => test_aws_storage_scripts.py} | 28 +++++++++++-------- 18 files changed, 89 insertions(+), 79 deletions(-) rename isvctl/configs/providers/aws/config/{block-storage.yaml => storage.yaml} (90%) rename isvctl/configs/providers/aws/scripts/{block-storage => storage}/create_volume.py (96%) rename isvctl/configs/providers/aws/scripts/{block-storage => storage}/snapshot_lifecycle.py (99%) rename isvctl/configs/providers/aws/scripts/{block-storage => storage}/teardown_volume.py (92%) rename isvctl/configs/providers/aws/scripts/{block-storage => storage}/volume_persistence.py (98%) rename isvctl/configs/providers/aws/scripts/{block-storage => storage}/volume_resize.py (95%) rename isvctl/configs/providers/my-isv/config/{block-storage.yaml => storage.yaml} (88%) rename isvctl/configs/providers/my-isv/scripts/{block-storage => storage}/create_volume.py (96%) rename isvctl/configs/providers/my-isv/scripts/{block-storage => storage}/snapshot_lifecycle.py (97%) rename isvctl/configs/providers/my-isv/scripts/{block-storage => storage}/teardown_volume.py (92%) rename isvctl/configs/providers/my-isv/scripts/{block-storage => storage}/volume_persistence.py (97%) rename isvctl/configs/providers/my-isv/scripts/{block-storage => storage}/volume_resize.py (97%) rename isvctl/tests/{test_aws_block_storage_scripts.py => test_aws_storage_scripts.py} (95%) diff --git a/Makefile b/Makefile index ce5ef621..af97b26a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -MY_ISV_DOMAINS := bare_metal block-storage control-plane iam image-registry network observability security vm +MY_ISV_DOMAINS := bare_metal control-plane iam image-registry network observability security storage vm DEMO_TARGETS := $(addprefix demo-,$(MY_ISV_DOMAINS)) .PHONY: help pre-commit build test coverage clean lint format install bump-patch bump-fix bump-minor bump-feat bump-major bump bump-check \ diff --git a/isvctl/configs/providers/aws/config/block-storage.yaml b/isvctl/configs/providers/aws/config/storage.yaml similarity index 90% rename from isvctl/configs/providers/aws/config/block-storage.yaml rename to isvctl/configs/providers/aws/config/storage.yaml index 892ac180..084fa715 100644 --- a/isvctl/configs/providers/aws/config/block-storage.yaml +++ b/isvctl/configs/providers/aws/config/storage.yaml @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# AWS Block Storage (EBS) Validation Configuration +# AWS Storage (EBS Block Volume) Validation Configuration # -# Block storage is the block-volume implementation of the storage suite, so +# Storage is the block-volume implementation of the storage suite, so # this config imports the provider-agnostic suites/storage.yaml and supplies # AWS-specific commands (boto3 + SSH) and settings. The launch_instance / # teardown steps reuse the VM reference scripts; the volume steps live in -# scripts/block-storage/. +# scripts/storage/. # # Steps: # SETUP @@ -34,7 +34,7 @@ # 7. teardown - boto3: terminate the instance + SG + key pair # # Usage: -# uv run isvctl test run -f isvctl/configs/providers/aws/config/block-storage.yaml +# uv run isvctl test run -f isvctl/configs/providers/aws/config/storage.yaml # # Required IAM Permissions: # ec2:DescribeInstances, ec2:RunInstances, ec2:TerminateInstances, @@ -70,7 +70,7 @@ commands: # AWS-specific: create + attach + format + mount + seed an EBS volume - name: create_volume phase: setup - command: "python3 ../scripts/block-storage/create_volume.py" + command: "python3 ../scripts/storage/create_volume.py" args: - "--instance-id" - "{{steps.launch_instance.instance_id}}" @@ -85,7 +85,7 @@ commands: # DATASVC-XX-02: snapshot -> restore -> verify data - name: snapshot_lifecycle phase: test - command: "python3 ../scripts/block-storage/snapshot_lifecycle.py" + command: "python3 ../scripts/storage/snapshot_lifecycle.py" args: - "--instance-id" - "{{steps.launch_instance.instance_id}}" @@ -102,7 +102,7 @@ commands: # DATASVC-XX-03: ModifyVolume + in-guest growpart/resize2fs - name: volume_resize phase: test - command: "python3 ../scripts/block-storage/volume_resize.py" + command: "python3 ../scripts/storage/volume_resize.py" args: - "--instance-id" - "{{steps.launch_instance.instance_id}}" @@ -120,7 +120,7 @@ commands: # Runs last so the stop/start IP churn does not affect the other tests. - name: volume_persistence phase: test - command: "python3 ../scripts/block-storage/volume_persistence.py" + command: "python3 ../scripts/storage/volume_persistence.py" args: - "--instance-id" - "{{steps.launch_instance.instance_id}}" @@ -139,7 +139,7 @@ commands: # AWS-specific: detach + delete the fixture volume - name: teardown_volume phase: teardown - command: "python3 ../scripts/block-storage/teardown_volume.py" + command: "python3 ../scripts/storage/teardown_volume.py" args: - "--region" - "{{region}}" @@ -163,7 +163,7 @@ commands: timeout: 600 tests: - cluster_name: "aws-block-storage-validation" + cluster_name: "aws-storage-validation" description: "AWS block storage (EBS) validation tests" settings: diff --git a/isvctl/configs/providers/aws/scripts/common/ebs.py b/isvctl/configs/providers/aws/scripts/common/ebs.py index 9e03277b..0874b697 100644 --- a/isvctl/configs/providers/aws/scripts/common/ebs.py +++ b/isvctl/configs/providers/aws/scripts/common/ebs.py @@ -16,7 +16,7 @@ """Shared EBS (block volume) helper utilities. Centralizes the boto3 volume / snapshot lifecycle used by the -block-storage validation scripts (create / attach / snapshot / restore / +storage validation scripts (create / attach / snapshot / restore / resize / detach / delete) plus the Nitro NVMe device-path mapping needed to find an attached volume from inside the guest. @@ -121,7 +121,7 @@ def create_snapshot( ec2: Any, volume_id: str, *, - description: str = "ISV block-storage validation snapshot", + description: str = "ISV storage validation snapshot", name: str = "isv-validate-snap", ) -> str: """Create a point-in-time snapshot of ``volume_id`` and return its ID.""" diff --git a/isvctl/configs/providers/aws/scripts/block-storage/create_volume.py b/isvctl/configs/providers/aws/scripts/storage/create_volume.py similarity index 96% rename from isvctl/configs/providers/aws/scripts/block-storage/create_volume.py rename to isvctl/configs/providers/aws/scripts/storage/create_volume.py index 0ea8addd..bed254ae 100644 --- a/isvctl/configs/providers/aws/scripts/block-storage/create_volume.py +++ b/isvctl/configs/providers/aws/scripts/storage/create_volume.py @@ -16,7 +16,7 @@ """Block-storage fixture: create, attach, format, mount, and seed a volume. -Shared setup step for the block-storage suite (DATASVC-XX-02/03/04). It +Shared setup step for the storage suite (DATASVC-XX-02/03/04). It creates an EBS volume in the instance's AZ, attaches it, partitions + formats it (ext4), mounts it, and writes a sentinel file. The volume ID, mount point, and sentinel content are passed to the snapshot / resize / @@ -25,14 +25,14 @@ Output JSON: { "success": true, - "platform": "block_storage", + "platform": "storage", "test_name": "create_volume", "volume_id": "vol-xxx", "device": "/dev/sdf", "mount_point": "/mnt/isv-block", "size_gib": 10, "sentinel_path": "/mnt/isv-block/isv-sentinel.txt", - "sentinel_content": "isv-ncp-validate-block-storage-...", + "sentinel_content": "isv-ncp-validate-storage-...", "operations": { "create": {"passed": true}, "attach": {"passed": true}, @@ -109,7 +109,7 @@ def main() -> int: parser.add_argument("--mount-point", default="/mnt/isv-block", help="In-guest mount point") args = parser.parse_args() - sentinel_content = f"isv-ncp-validate-block-storage-{uuid.uuid4().hex}" + sentinel_content = f"isv-ncp-validate-storage-{uuid.uuid4().hex}" operations: dict[str, dict[str, Any]] = { "create": {"passed": False}, "attach": {"passed": False}, @@ -119,7 +119,7 @@ def main() -> int: } result: dict[str, Any] = { "success": False, - "platform": "block_storage", + "platform": "storage", "test_name": "create_volume", "instance_id": args.instance_id, "volume_id": None, diff --git a/isvctl/configs/providers/aws/scripts/block-storage/snapshot_lifecycle.py b/isvctl/configs/providers/aws/scripts/storage/snapshot_lifecycle.py similarity index 99% rename from isvctl/configs/providers/aws/scripts/block-storage/snapshot_lifecycle.py rename to isvctl/configs/providers/aws/scripts/storage/snapshot_lifecycle.py index 51740111..ded557ed 100644 --- a/isvctl/configs/providers/aws/scripts/block-storage/snapshot_lifecycle.py +++ b/isvctl/configs/providers/aws/scripts/storage/snapshot_lifecycle.py @@ -25,7 +25,7 @@ Output JSON: { "success": true, - "platform": "block_storage", + "platform": "storage", "test_name": "snapshot_lifecycle", "volume_id": "vol-source", "snapshot_id": "snap-xxx", @@ -79,7 +79,7 @@ def main() -> int: } result: dict[str, Any] = { "success": False, - "platform": "block_storage", + "platform": "storage", "test_name": "snapshot_lifecycle", "volume_id": args.volume_id, "snapshot_id": None, diff --git a/isvctl/configs/providers/aws/scripts/block-storage/teardown_volume.py b/isvctl/configs/providers/aws/scripts/storage/teardown_volume.py similarity index 92% rename from isvctl/configs/providers/aws/scripts/block-storage/teardown_volume.py rename to isvctl/configs/providers/aws/scripts/storage/teardown_volume.py index ba9d1162..d91c8c58 100644 --- a/isvctl/configs/providers/aws/scripts/block-storage/teardown_volume.py +++ b/isvctl/configs/providers/aws/scripts/storage/teardown_volume.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Detach and delete the block-storage fixture volume (teardown). +"""Detach and delete the storage fixture volume (teardown). Best-effort cleanup of the volume created by ``create_volume.py``. Runs before the instance teardown so the volume is removed explicitly rather @@ -24,7 +24,7 @@ Output JSON: { "success": true, - "platform": "block_storage", + "platform": "storage", "test_name": "teardown_volume", "resources_deleted": ["vol-xxx"] } @@ -45,7 +45,7 @@ def main() -> int: """Detach and delete the fixture volume; print JSON result.""" - parser = argparse.ArgumentParser(description="Teardown block-storage fixture volume") + parser = argparse.ArgumentParser(description="Teardown storage fixture volume") parser.add_argument("--region", default=os.environ.get("AWS_REGION", "us-west-2")) parser.add_argument("--volume-id", required=True, help="Fixture volume to delete") parser.add_argument("--skip-destroy", action="store_true", help="Skip actual destroy") @@ -53,7 +53,7 @@ def main() -> int: result: dict[str, Any] = { "success": False, - "platform": "block_storage", + "platform": "storage", "test_name": "teardown_volume", "resources_deleted": [], } diff --git a/isvctl/configs/providers/aws/scripts/block-storage/volume_persistence.py b/isvctl/configs/providers/aws/scripts/storage/volume_persistence.py similarity index 98% rename from isvctl/configs/providers/aws/scripts/block-storage/volume_persistence.py rename to isvctl/configs/providers/aws/scripts/storage/volume_persistence.py index 3d8be1f3..df366c40 100644 --- a/isvctl/configs/providers/aws/scripts/block-storage/volume_persistence.py +++ b/isvctl/configs/providers/aws/scripts/storage/volume_persistence.py @@ -24,7 +24,7 @@ Output JSON: { "success": true, - "platform": "block_storage", + "platform": "storage", "test_name": "volume_persistence", "volume_id": "vol-xxx", "operations": { @@ -78,7 +78,7 @@ def main() -> int: } result: dict[str, Any] = { "success": False, - "platform": "block_storage", + "platform": "storage", "test_name": "volume_persistence", "volume_id": args.volume_id, "instance_id": args.instance_id, diff --git a/isvctl/configs/providers/aws/scripts/block-storage/volume_resize.py b/isvctl/configs/providers/aws/scripts/storage/volume_resize.py similarity index 95% rename from isvctl/configs/providers/aws/scripts/block-storage/volume_resize.py rename to isvctl/configs/providers/aws/scripts/storage/volume_resize.py index 53906258..4d325604 100644 --- a/isvctl/configs/providers/aws/scripts/block-storage/volume_resize.py +++ b/isvctl/configs/providers/aws/scripts/storage/volume_resize.py @@ -24,7 +24,7 @@ Output JSON: { "success": true, - "platform": "block_storage", + "platform": "storage", "test_name": "volume_resize", "volume_id": "vol-xxx", "old_size_gib": 10, @@ -101,7 +101,7 @@ def main() -> int: } result: dict[str, Any] = { "success": False, - "platform": "block_storage", + "platform": "storage", "test_name": "volume_resize", "volume_id": args.volume_id, "operations": operations, @@ -139,7 +139,9 @@ def main() -> int: print(json.dumps(result, indent=2)) return 1 - rc, before_out, _ = ssh_run(public_ip, args.ssh_user, args.key_file, _READ_FS_BYTES.replace("__MOUNT__", args.mount_point)) + rc, before_out, _ = ssh_run( + public_ip, args.ssh_user, args.key_file, _READ_FS_BYTES.replace("__MOUNT__", args.mount_point) + ) fs_before = int(before_out.strip()) if rc == 0 and before_out.strip().isdigit() else None result["fs_bytes_before"] = fs_before @@ -156,7 +158,9 @@ def main() -> int: else: _fail(operations["resize_filesystem"], f"resize2fs failed (rc={rc}): {err.strip()[:300]}") - rc, after_out, _ = ssh_run(public_ip, args.ssh_user, args.key_file, _READ_FS_BYTES.replace("__MOUNT__", args.mount_point)) + rc, after_out, _ = ssh_run( + public_ip, args.ssh_user, args.key_file, _READ_FS_BYTES.replace("__MOUNT__", args.mount_point) + ) fs_after = int(after_out.strip()) if rc == 0 and after_out.strip().isdigit() else None result["fs_bytes_after"] = fs_after diff --git a/isvctl/configs/providers/my-isv/config/block-storage.yaml b/isvctl/configs/providers/my-isv/config/storage.yaml similarity index 88% rename from isvctl/configs/providers/my-isv/config/block-storage.yaml rename to isvctl/configs/providers/my-isv/config/storage.yaml index 28ab6907..af1d5fa3 100644 --- a/isvctl/configs/providers/my-isv/config/block-storage.yaml +++ b/isvctl/configs/providers/my-isv/config/storage.yaml @@ -13,16 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -# my-isv Block Storage Validation Configuration - Living Example +# my-isv Storage Validation Configuration - Living Example # -# Block storage is the block-volume implementation of the storage suite, so +# Storage is the block-volume implementation of the storage suite, so # this config imports the provider-agnostic suites/storage.yaml and wires the # commands to the generic template scripts. Those scripts ship with # dummy-success values so this example runs end-to-end without any real cloud # infrastructure (ISVCTL_DEMO_MODE=1, used by `make demo-test`). # # The launch_instance / teardown steps reuse the VM scaffold scripts; the -# volume steps live in scripts/block-storage/. +# volume steps live in scripts/storage/. # # Steps: # SETUP @@ -37,7 +37,7 @@ # 7. teardown - Terminate the instance # # Usage: -# uv run isvctl test run -f isvctl/configs/providers/my-isv/config/block-storage.yaml +# uv run isvctl test run -f isvctl/configs/providers/my-isv/config/storage.yaml import: - ../../../suites/storage.yaml @@ -62,7 +62,7 @@ commands: - name: create_volume phase: setup - command: "python ../scripts/block-storage/create_volume.py" + command: "python ../scripts/storage/create_volume.py" args: - "--instance-id" - "{{steps.launch_instance.instance_id}}" @@ -76,7 +76,7 @@ commands: - name: snapshot_lifecycle phase: test - command: "python ../scripts/block-storage/snapshot_lifecycle.py" + command: "python ../scripts/storage/snapshot_lifecycle.py" args: - "--instance-id" - "{{steps.launch_instance.instance_id}}" @@ -92,7 +92,7 @@ commands: - name: volume_resize phase: test - command: "python ../scripts/block-storage/volume_resize.py" + command: "python ../scripts/storage/volume_resize.py" args: - "--instance-id" - "{{steps.launch_instance.instance_id}}" @@ -108,7 +108,7 @@ commands: - name: volume_persistence phase: test - command: "python ../scripts/block-storage/volume_persistence.py" + command: "python ../scripts/storage/volume_persistence.py" args: - "--instance-id" - "{{steps.launch_instance.instance_id}}" @@ -126,7 +126,7 @@ commands: - name: teardown_volume phase: teardown - command: "python ../scripts/block-storage/teardown_volume.py" + command: "python ../scripts/storage/teardown_volume.py" args: - "--region" - "{{region}}" @@ -148,7 +148,7 @@ commands: timeout: 60 tests: - cluster_name: "my-isv-block-storage-validation" + cluster_name: "my-isv-storage-validation" description: "my-isv block storage validation (generic stubs, dummy overrides)" settings: diff --git a/isvctl/configs/providers/my-isv/scripts/README.md b/isvctl/configs/providers/my-isv/scripts/README.md index f614bca6..43634c2c 100644 --- a/isvctl/configs/providers/my-isv/scripts/README.md +++ b/isvctl/configs/providers/my-isv/scripts/README.md @@ -31,7 +31,7 @@ template, then fill in the TODOs. | `control-plane/` | 10 | [`suites/control-plane.yaml`](../../../suites/control-plane.yaml) | [`config/control-plane.yaml`](../config/control-plane.yaml) | [`providers/aws/scripts/control-plane/`](../../aws/scripts/control-plane/) | | `vm/` | 9 | [`suites/vm.yaml`](../../../suites/vm.yaml) | [`config/vm.yaml`](../config/vm.yaml) | [`providers/aws/scripts/vm/`](../../aws/scripts/vm/) | | `bare_metal/` | 12 | [`suites/bare_metal.yaml`](../../../suites/bare_metal.yaml) | [`config/bare_metal.yaml`](../config/bare_metal.yaml) | [`providers/aws/scripts/bare_metal/`](../../aws/scripts/bare_metal/) | -| `block-storage/` | 5 | [`suites/storage.yaml`](../../../suites/storage.yaml) | [`config/block-storage.yaml`](../config/block-storage.yaml) | [`providers/aws/scripts/block-storage/`](../../aws/scripts/block-storage/) | +| `storage/` | 5 | [`suites/storage.yaml`](../../../suites/storage.yaml) | [`config/storage.yaml`](../config/storage.yaml) | [`providers/aws/scripts/storage/`](../../aws/scripts/storage/) | | `network/` | 18 | [`suites/network.yaml`](../../../suites/network.yaml) | [`config/network.yaml`](../config/network.yaml) | [`providers/aws/scripts/network/`](../../aws/scripts/network/) | | `observability/` | 1 | [`suites/observability.yaml`](../../../suites/observability.yaml) | [`config/observability.yaml`](../config/observability.yaml) | [`providers/aws/scripts/observability/`](../../aws/scripts/observability/) | | `image-registry/` | 7 | [`suites/image-registry.yaml`](../../../suites/image-registry.yaml) | [`config/image-registry.yaml`](../config/image-registry.yaml) | [`providers/aws/scripts/image-registry/`](../../aws/scripts/image-registry/) | diff --git a/isvctl/configs/providers/my-isv/scripts/block-storage/create_volume.py b/isvctl/configs/providers/my-isv/scripts/storage/create_volume.py similarity index 96% rename from isvctl/configs/providers/my-isv/scripts/block-storage/create_volume.py rename to isvctl/configs/providers/my-isv/scripts/storage/create_volume.py index de7e8707..e04b96d6 100644 --- a/isvctl/configs/providers/my-isv/scripts/block-storage/create_volume.py +++ b/isvctl/configs/providers/my-isv/scripts/storage/create_volume.py @@ -23,7 +23,7 @@ Required JSON output fields: success (bool) - true iff every operation passed - platform (str) - "block_storage" + platform (str) - "storage" test_name (str) - "create_volume" volume_id (str) - identifier of the created volume (consumed by later steps) mount_point (str) - where the volume is mounted in the guest @@ -40,7 +40,7 @@ python create_volume.py --instance-id --region --key-file --size-gib 10 Reference implementation (AWS): - ../aws/block-storage/create_volume.py + ../aws/storage/create_volume.py """ import argparse @@ -64,7 +64,7 @@ def main() -> int: parser.add_argument("--mount-point", default="/mnt/isv-block", help="In-guest mount point") args = parser.parse_args() - sentinel_content = f"isv-ncp-validate-block-storage-{uuid.uuid4().hex}" + sentinel_content = f"isv-ncp-validate-storage-{uuid.uuid4().hex}" operations: dict[str, dict[str, Any]] = { "create": {"passed": False}, "attach": {"passed": False}, @@ -74,7 +74,7 @@ def main() -> int: } result: dict[str, Any] = { "success": False, - "platform": "block_storage", + "platform": "storage", "test_name": "create_volume", "volume_id": "", "mount_point": args.mount_point, diff --git a/isvctl/configs/providers/my-isv/scripts/block-storage/snapshot_lifecycle.py b/isvctl/configs/providers/my-isv/scripts/storage/snapshot_lifecycle.py similarity index 97% rename from isvctl/configs/providers/my-isv/scripts/block-storage/snapshot_lifecycle.py rename to isvctl/configs/providers/my-isv/scripts/storage/snapshot_lifecycle.py index 9baed6ba..06ede89f 100644 --- a/isvctl/configs/providers/my-isv/scripts/block-storage/snapshot_lifecycle.py +++ b/isvctl/configs/providers/my-isv/scripts/storage/snapshot_lifecycle.py @@ -24,7 +24,7 @@ Required JSON output fields: success (bool) - true iff every operation passed - platform (str) - "block_storage" + platform (str) - "storage" test_name (str) - "snapshot_lifecycle" volume_id (str) - source (fixture) volume that was snapshotted snapshot_id (str) - identifier of the created snapshot @@ -38,7 +38,7 @@ python snapshot_lifecycle.py --volume-id --expected-content Reference implementation (AWS): - ../aws/block-storage/snapshot_lifecycle.py + ../aws/storage/snapshot_lifecycle.py """ import argparse @@ -69,7 +69,7 @@ def main() -> int: } result: dict[str, Any] = { "success": False, - "platform": "block_storage", + "platform": "storage", "test_name": "snapshot_lifecycle", "volume_id": args.volume_id, "snapshot_id": "", diff --git a/isvctl/configs/providers/my-isv/scripts/block-storage/teardown_volume.py b/isvctl/configs/providers/my-isv/scripts/storage/teardown_volume.py similarity index 92% rename from isvctl/configs/providers/my-isv/scripts/block-storage/teardown_volume.py rename to isvctl/configs/providers/my-isv/scripts/storage/teardown_volume.py index 04607534..4633e746 100644 --- a/isvctl/configs/providers/my-isv/scripts/block-storage/teardown_volume.py +++ b/isvctl/configs/providers/my-isv/scripts/storage/teardown_volume.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Detach and delete the block-storage fixture volume (teardown). +"""Detach and delete the storage fixture volume (teardown). Provider-agnostic template - replace the TODO block with your platform's detach + delete API calls for the fixture volume created by @@ -22,7 +22,7 @@ Required JSON output fields: success (bool) - whether cleanup succeeded - platform (str) - "block_storage" + platform (str) - "storage" test_name (str) - "teardown_volume" resources_deleted (list) - identifiers of volumes that were deleted message (str) - human-readable summary @@ -31,7 +31,7 @@ python teardown_volume.py --volume-id --region Reference implementation (AWS): - ../aws/block-storage/teardown_volume.py + ../aws/storage/teardown_volume.py """ import argparse @@ -46,7 +46,7 @@ def main() -> int: """Detach and delete the fixture volume; print JSON result.""" - parser = argparse.ArgumentParser(description="Teardown block-storage fixture volume") + parser = argparse.ArgumentParser(description="Teardown storage fixture volume") parser.add_argument("--region", default="", help="Cloud region") parser.add_argument("--volume-id", default="", help="Fixture volume to delete") parser.add_argument("--skip-destroy", action="store_true", help="Skip actual destroy") @@ -54,7 +54,7 @@ def main() -> int: result: dict[str, Any] = { "success": False, - "platform": "block_storage", + "platform": "storage", "test_name": "teardown_volume", "resources_deleted": [], "message": "", diff --git a/isvctl/configs/providers/my-isv/scripts/block-storage/volume_persistence.py b/isvctl/configs/providers/my-isv/scripts/storage/volume_persistence.py similarity index 97% rename from isvctl/configs/providers/my-isv/scripts/block-storage/volume_persistence.py rename to isvctl/configs/providers/my-isv/scripts/storage/volume_persistence.py index f91a8eb9..d760c3d4 100644 --- a/isvctl/configs/providers/my-isv/scripts/block-storage/volume_persistence.py +++ b/isvctl/configs/providers/my-isv/scripts/storage/volume_persistence.py @@ -23,7 +23,7 @@ Required JSON output fields: success (bool) - true iff every operation passed - platform (str) - "block_storage" + platform (str) - "storage" test_name (str) - "volume_persistence" volume_id (str) - volume expected to persist operations: { @@ -37,7 +37,7 @@ python volume_persistence.py --instance-id --volume-id --expected-content Reference implementation (AWS): - ../aws/block-storage/volume_persistence.py + ../aws/storage/volume_persistence.py """ import argparse @@ -69,7 +69,7 @@ def main() -> int: } result: dict[str, Any] = { "success": False, - "platform": "block_storage", + "platform": "storage", "test_name": "volume_persistence", "volume_id": args.volume_id, "instance_id": args.instance_id, diff --git a/isvctl/configs/providers/my-isv/scripts/block-storage/volume_resize.py b/isvctl/configs/providers/my-isv/scripts/storage/volume_resize.py similarity index 97% rename from isvctl/configs/providers/my-isv/scripts/block-storage/volume_resize.py rename to isvctl/configs/providers/my-isv/scripts/storage/volume_resize.py index a5b3f0c4..65675f16 100644 --- a/isvctl/configs/providers/my-isv/scripts/block-storage/volume_resize.py +++ b/isvctl/configs/providers/my-isv/scripts/storage/volume_resize.py @@ -22,7 +22,7 @@ Required JSON output fields: success (bool) - true iff every operation passed - platform (str) - "block_storage" + platform (str) - "storage" test_name (str) - "volume_resize" volume_id (str) - volume that was resized operations: { @@ -36,7 +36,7 @@ python volume_resize.py --volume-id --mount-point /mnt/isv-block Reference implementation (AWS): - ../aws/block-storage/volume_resize.py + ../aws/storage/volume_resize.py """ import argparse @@ -68,7 +68,7 @@ def main() -> int: } result: dict[str, Any] = { "success": False, - "platform": "block_storage", + "platform": "storage", "test_name": "volume_resize", "volume_id": args.volume_id, "operations": operations, diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index 08c2680f..aadec5d5 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -133,11 +133,11 @@ volume. The three test-phase steps all reuse that fixture. | Step | Phase | Script | Key JSON Fields | |------|-------|--------|-----------------| | `launch_instance` | setup | `providers/my-isv/scripts/vm/launch_instance.py` | `instance_id`, `state`, `public_ip`, `key_file` (reuses VM script) | -| `create_volume` | setup | `providers/my-isv/scripts/block-storage/create_volume.py` | `volume_id`, `mount_point`, `sentinel_content`, `operations.{create,attach,format,mount,write_sentinel}` | -| `snapshot_lifecycle` | test | `providers/my-isv/scripts/block-storage/snapshot_lifecycle.py` | `volume_id`, `snapshot_id`, `operations.{create_snapshot,restore_volume,verify_data}` (verify_data includes `content_matches`) | -| `volume_resize` | test | `providers/my-isv/scripts/block-storage/volume_resize.py` | `volume_id`, `operations.{modify_volume,grow_partition,resize_filesystem,verify_size}` | -| `volume_persistence` | test | `providers/my-isv/scripts/block-storage/volume_persistence.py` | `volume_id`, `operations.{stop,start,verify_attached,verify_data}` (verify_data includes `content_matches`) | -| `teardown_volume` | teardown | `providers/my-isv/scripts/block-storage/teardown_volume.py` | `resources_deleted`, `message` | +| `create_volume` | setup | `providers/my-isv/scripts/storage/create_volume.py` | `volume_id`, `mount_point`, `sentinel_content`, `operations.{create,attach,format,mount,write_sentinel}` | +| `snapshot_lifecycle` | test | `providers/my-isv/scripts/storage/snapshot_lifecycle.py` | `volume_id`, `snapshot_id`, `operations.{create_snapshot,restore_volume,verify_data}` (verify_data includes `content_matches`) | +| `volume_resize` | test | `providers/my-isv/scripts/storage/volume_resize.py` | `volume_id`, `operations.{modify_volume,grow_partition,resize_filesystem,verify_size}` | +| `volume_persistence` | test | `providers/my-isv/scripts/storage/volume_persistence.py` | `volume_id`, `operations.{stop,start,verify_attached,verify_data}` (verify_data includes `content_matches`) | +| `teardown_volume` | teardown | `providers/my-isv/scripts/storage/teardown_volume.py` | `resources_deleted`, `message` | | `teardown` | teardown | `providers/my-isv/scripts/vm/teardown.py` | `resources_deleted`, `message` (reuses VM script) | ### Kubernetes (`k8s.yaml`) diff --git a/isvctl/configs/suites/storage.yaml b/isvctl/configs/suites/storage.yaml index ce92d14e..476229d1 100644 --- a/isvctl/configs/suites/storage.yaml +++ b/isvctl/configs/suites/storage.yaml @@ -40,13 +40,13 @@ # instance and confirm the volume stays attached with its data intact. # # Quick start: -# 1. Copy providers/my-isv/config/block-storage.yaml to providers//config/block-storage.yaml +# 1. Copy providers/my-isv/config/storage.yaml to providers//config/storage.yaml # 2. Replace stub paths with your platform scripts -# 3. Run: uv run isvctl test run -f isvctl/configs/providers//config/block-storage.yaml +# 3. Run: uv run isvctl test run -f isvctl/configs/providers//config/storage.yaml # # See also: -# - my-isv living example: providers/my-isv/config/block-storage.yaml -# - AWS reference: providers/aws/config/block-storage.yaml + providers/aws/scripts/block-storage/ +# - my-isv living example: providers/my-isv/config/storage.yaml +# - AWS reference: providers/aws/config/storage.yaml + providers/aws/scripts/storage/ # - Per-step field breakdown: ../suites/README.md version: "1.0" diff --git a/isvctl/tests/test_aws_block_storage_scripts.py b/isvctl/tests/test_aws_storage_scripts.py similarity index 95% rename from isvctl/tests/test_aws_block_storage_scripts.py rename to isvctl/tests/test_aws_storage_scripts.py index 1f7d007b..8fc24e7e 100644 --- a/isvctl/tests/test_aws_block_storage_scripts.py +++ b/isvctl/tests/test_aws_storage_scripts.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for AWS block-storage reference scripts (DATASVC-XX-02/03/04).""" +"""Tests for AWS storage reference scripts (DATASVC-XX-02/03/04).""" from __future__ import annotations @@ -28,15 +28,15 @@ from botocore.exceptions import ClientError ISVCTL_ROOT = Path(__file__).resolve().parents[1] -AWS_BLOCK_STORAGE_SCRIPTS = ISVCTL_ROOT / "configs" / "providers" / "aws" / "scripts" / "block-storage" +AWS_STORAGE_SCRIPTS = ISVCTL_ROOT / "configs" / "providers" / "aws" / "scripts" / "storage" -EXPECTED_CONTENT = "isv-ncp-validate-block-storage-deadbeef" +EXPECTED_CONTENT = "isv-ncp-validate-storage-deadbeef" def _load_script(script_name: str) -> ModuleType: - """Load an AWS block-storage script as a module for direct testing.""" - script_path = AWS_BLOCK_STORAGE_SCRIPTS / script_name - spec = importlib.util.spec_from_file_location(f"test_bs_{script_path.stem}", script_path) + """Load an AWS storage script as a module for direct testing.""" + script_path = AWS_STORAGE_SCRIPTS / script_name + spec = importlib.util.spec_from_file_location(f"test_storage_{script_path.stem}", script_path) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -224,9 +224,7 @@ def test_create_volume_guest_setup_failure_fails_step( # --------------------------------------------------------------------------- -def test_snapshot_lifecycle_happy_path( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: +def test_snapshot_lifecycle_happy_path(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: """A matching restored sentinel passes the snapshot round-trip.""" module = _load_script("snapshot_lifecycle.py") monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FakeEc2()) @@ -359,7 +357,11 @@ def fake_ssh(host: str, user: str, key: str, script: str, **kwargs: Any) -> tupl return (0, "", "") monkeypatch.setattr(module, "ssh_run", fake_ssh) - monkeypatch.setattr(sys, "argv", ["volume_resize.py", "--instance-id", "i-fixture", "--volume-id", "vol-fixture", "--key-file", "/tmp/k.pem"]) + monkeypatch.setattr( + sys, + "argv", + ["volume_resize.py", "--instance-id", "i-fixture", "--volume-id", "vol-fixture", "--key-file", "/tmp/k.pem"], + ) exit_code = module.main() payload = json.loads(capsys.readouterr().out) @@ -387,7 +389,11 @@ def fake_ssh(host: str, user: str, key: str, script: str, **kwargs: Any) -> tupl return (0, "", "") monkeypatch.setattr(module, "ssh_run", fake_ssh) - monkeypatch.setattr(sys, "argv", ["volume_resize.py", "--instance-id", "i-fixture", "--volume-id", "vol-fixture", "--key-file", "/tmp/k.pem"]) + monkeypatch.setattr( + sys, + "argv", + ["volume_resize.py", "--instance-id", "i-fixture", "--volume-id", "vol-fixture", "--key-file", "/tmp/k.pem"], + ) exit_code = module.main() payload = json.loads(capsys.readouterr().out) From cfe0ce891bb0f6dfe97db860ddae2d0215903b45 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 9 Jul 2026 18:30:42 +0000 Subject: [PATCH 09/10] fix(storage): add suite wiring metadata for CI guardrails Add test_id and labels to storage.yaml checks (DATASVC02-01 through DATASVC04-01), register the storage suite in validate_suite_wiring.py, and sync the storage label into docs/test-plan.yaml. These guardrails landed on main after PR #439 was opened. Signed-off-by: Cursor Agent Co-authored-by: Alexandre Begnoche --- docs/test-plan.yaml | 3 +++ isvctl/configs/suites/storage.yaml | 38 ++++++++++++++++++++++++++---- scripts/validate_suite_wiring.py | 1 + 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/docs/test-plan.yaml b/docs/test-plan.yaml index 94c787ff..8451c345 100644 --- a/docs/test-plan.yaml +++ b/docs/test-plan.yaml @@ -2444,6 +2444,7 @@ domains: - summary: Verify volume snapshots labels: - min_req + - storage priority: P1 milestone: M5 notes: "" @@ -2456,6 +2457,7 @@ domains: - summary: Verify volume resizing labels: - min_req + - storage priority: P1 milestone: M5 notes: "" @@ -2468,6 +2470,7 @@ domains: - summary: Verify persistent block volumes survive instance restarts labels: - min_req + - storage priority: P1 milestone: M5 notes: "" diff --git a/isvctl/configs/suites/storage.yaml b/isvctl/configs/suites/storage.yaml index 476229d1..6769580f 100644 --- a/isvctl/configs/suites/storage.yaml +++ b/isvctl/configs/suites/storage.yaml @@ -71,52 +71,80 @@ tests: step: launch_instance checks: InstanceStateCheck: + test_id: "N/A" + labels: ["storage"] expected_state: "running" fixture_volume: step: create_volume checks: - StepSuccessCheck: {} + StepSuccessCheck: + test_id: "N/A" + labels: ["storage"] FieldExistsCheck: + test_id: "N/A" + labels: ["storage"] fields: ["volume_id", "mount_point", "operations"] CrudOperationsCheck: + test_id: "N/A" + labels: ["storage"] operations: ["create", "attach", "format", "mount", "write_sentinel"] # ─── DATASVC-XX-02: volume snapshots (issue #321) ──────────────── snapshot_lifecycle: step: snapshot_lifecycle checks: - StepSuccessCheck: {} + StepSuccessCheck: + test_id: "N/A" + labels: ["storage"] FieldExistsCheck: + test_id: "N/A" + labels: ["storage"] fields: ["volume_id", "snapshot_id", "operations"] CrudOperationsCheck: + test_id: "DATASVC02-01" + labels: ["min_req", "storage"] operations: ["create_snapshot", "restore_volume", "verify_data"] # ─── DATASVC-XX-03: volume resizing (issue #322) ───────────────── volume_resize: step: volume_resize checks: - StepSuccessCheck: {} + StepSuccessCheck: + test_id: "N/A" + labels: ["storage"] FieldExistsCheck: + test_id: "N/A" + labels: ["storage"] fields: ["volume_id", "operations"] CrudOperationsCheck: + test_id: "DATASVC03-01" + labels: ["min_req", "storage"] operations: ["modify_volume", "grow_partition", "resize_filesystem", "verify_size"] # ─── DATASVC-XX-04: persistence across restarts (issue #323) ───── volume_persistence: step: volume_persistence checks: - StepSuccessCheck: {} + StepSuccessCheck: + test_id: "N/A" + labels: ["storage"] FieldExistsCheck: + test_id: "N/A" + labels: ["storage"] fields: ["volume_id", "operations"] CrudOperationsCheck: + test_id: "DATASVC04-01" + labels: ["min_req", "storage"] operations: ["stop", "start", "verify_attached", "verify_data"] # ─── Teardown ──────────────────────────────────────────────────── teardown_checks: step: teardown_volume checks: - StepSuccessCheck: {} + StepSuccessCheck: + test_id: "N/A" + labels: ["storage"] exclude: labels: [] diff --git a/scripts/validate_suite_wiring.py b/scripts/validate_suite_wiring.py index 5837be79..fc4c5066 100644 --- a/scripts/validate_suite_wiring.py +++ b/scripts/validate_suite_wiring.py @@ -55,6 +55,7 @@ "observability": "observability", "security": "security", "slurm": "slurm", + "storage": "storage", "vm": "vm", } From 798f7881940eb9e02252a043cc38c72f717341b6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 9 Jul 2026 18:45:29 +0000 Subject: [PATCH 10/10] fix(storage): harden AWS scripts against empty/missing instance lookups Add ebs.describe_instance() and use it in all four AWS storage scripts so an empty DescribeInstances response returns JSON instead of IndexError. Wrap the final describe_volumes size check in volume_resize.py, preserve API error messages when verify_size already failed, and reject non-positive --grow-gib in the my-isv demo stub. Signed-off-by: Cursor Agent Co-authored-by: Alexandre Begnoche --- .../providers/aws/scripts/common/ebs.py | 14 ++++ .../aws/scripts/storage/create_volume.py | 5 +- .../aws/scripts/storage/snapshot_lifecycle.py | 5 +- .../aws/scripts/storage/volume_persistence.py | 5 +- .../aws/scripts/storage/volume_resize.py | 14 ++-- .../my-isv/scripts/storage/volume_resize.py | 3 + isvctl/tests/test_aws_storage_scripts.py | 77 +++++++++++++++++++ 7 files changed, 109 insertions(+), 14 deletions(-) diff --git a/isvctl/configs/providers/aws/scripts/common/ebs.py b/isvctl/configs/providers/aws/scripts/common/ebs.py index 0874b697..9126aa51 100644 --- a/isvctl/configs/providers/aws/scripts/common/ebs.py +++ b/isvctl/configs/providers/aws/scripts/common/ebs.py @@ -58,6 +58,20 @@ def guest_by_id_path(volume_id: str) -> str: return f"/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_{nvme_serial_for_volume(volume_id)}" +def describe_instance(ec2: Any, instance_id: str) -> dict[str, Any]: + """Return the instance dict for ``instance_id``. + + Raises: + RuntimeError: When ``describe_instances`` returns no matching instance + (for example after termination, when the ID is stale). + """ + response = ec2.describe_instances(InstanceIds=[instance_id]) + reservations = response.get("Reservations", []) + if not reservations or not reservations[0].get("Instances"): + raise RuntimeError(f"Instance not found: {instance_id}") + return reservations[0]["Instances"][0] + + def _tag_spec(resource_type: str, name: str) -> dict[str, Any]: """Build a TagSpecifications entry carrying the Name and ownership tags.""" return { diff --git a/isvctl/configs/providers/aws/scripts/storage/create_volume.py b/isvctl/configs/providers/aws/scripts/storage/create_volume.py index bed254ae..bda8960b 100644 --- a/isvctl/configs/providers/aws/scripts/storage/create_volume.py +++ b/isvctl/configs/providers/aws/scripts/storage/create_volume.py @@ -134,8 +134,7 @@ def main() -> int: ec2 = boto3.client("ec2", region_name=args.region) try: - instances = ec2.describe_instances(InstanceIds=[args.instance_id]) - instance = instances["Reservations"][0]["Instances"][0] + instance = ebs.describe_instance(ec2, args.instance_id) availability_zone = instance["Placement"]["AvailabilityZone"] public_ip = instance.get("PublicIpAddress") result["availability_zone"] = availability_zone @@ -148,7 +147,7 @@ def main() -> int: ebs.attach_volume(ec2, volume_id, args.instance_id, args.device) ebs.wait_for_volume_in_use(ec2, volume_id) operations["attach"]["passed"] = True - except (ClientError, BotoCoreError) as e: + except (ClientError, BotoCoreError, RuntimeError) as e: result["error"] = f"Volume create/attach failed: {e}" print(json.dumps(result, indent=2)) return 1 diff --git a/isvctl/configs/providers/aws/scripts/storage/snapshot_lifecycle.py b/isvctl/configs/providers/aws/scripts/storage/snapshot_lifecycle.py index ded557ed..398c86e2 100644 --- a/isvctl/configs/providers/aws/scripts/storage/snapshot_lifecycle.py +++ b/isvctl/configs/providers/aws/scripts/storage/snapshot_lifecycle.py @@ -92,8 +92,7 @@ def main() -> int: snapshot_id: str | None = None restored_volume_id: str | None = None try: - instances = ec2.describe_instances(InstanceIds=[args.instance_id]) - instance = instances["Reservations"][0]["Instances"][0] + instance = ebs.describe_instance(ec2, args.instance_id) availability_zone = instance["Placement"]["AvailabilityZone"] public_ip = instance.get("PublicIpAddress") @@ -151,7 +150,7 @@ def main() -> int: result["success"] = all(op["passed"] for op in operations.values()) return _emit(result, ec2, restored_volume_id, snapshot_id) - except (ClientError, BotoCoreError) as e: + except (ClientError, BotoCoreError, RuntimeError) as e: result["error"] = str(e) return _emit(result, ec2, restored_volume_id, snapshot_id) diff --git a/isvctl/configs/providers/aws/scripts/storage/volume_persistence.py b/isvctl/configs/providers/aws/scripts/storage/volume_persistence.py index df366c40..7fb0d931 100644 --- a/isvctl/configs/providers/aws/scripts/storage/volume_persistence.py +++ b/isvctl/configs/providers/aws/scripts/storage/volume_persistence.py @@ -98,8 +98,7 @@ def main() -> int: ec2.get_waiter("instance_status_ok").wait( InstanceIds=[args.instance_id], WaiterConfig={"Delay": 15, "MaxAttempts": 40} ) - instances = ec2.describe_instances(InstanceIds=[args.instance_id]) - instance = instances["Reservations"][0]["Instances"][0] + instance = ebs.describe_instance(ec2, args.instance_id) public_ip = instance.get("PublicIpAddress") or wait_for_public_ip(ec2, args.instance_id) if not public_ip: _fail(operations["start"], "No public IP after restart") @@ -112,7 +111,7 @@ def main() -> int: print(json.dumps(result, indent=2)) return 1 operations["start"]["passed"] = True - except (ClientError, BotoCoreError) as e: + except (ClientError, BotoCoreError, RuntimeError) as e: result["error"] = f"Restart failed: {e}" print(json.dumps(result, indent=2)) return 1 diff --git a/isvctl/configs/providers/aws/scripts/storage/volume_resize.py b/isvctl/configs/providers/aws/scripts/storage/volume_resize.py index 4d325604..5c5619a7 100644 --- a/isvctl/configs/providers/aws/scripts/storage/volume_resize.py +++ b/isvctl/configs/providers/aws/scripts/storage/volume_resize.py @@ -111,8 +111,8 @@ def main() -> int: by_id = ebs.guest_by_id_path(args.volume_id) try: - instances = ec2.describe_instances(InstanceIds=[args.instance_id]) - public_ip = instances["Reservations"][0]["Instances"][0].get("PublicIpAddress") + instance = ebs.describe_instance(ec2, args.instance_id) + public_ip = instance.get("PublicIpAddress") volumes = ec2.describe_volumes(VolumeIds=[args.volume_id]) old_size = volumes["Volumes"][0]["Size"] @@ -129,7 +129,7 @@ def main() -> int: result["error"] = f"ModifyVolume failed: {e}" print(json.dumps(result, indent=2)) return 1 - except (ClientError, BotoCoreError) as e: + except (ClientError, BotoCoreError, RuntimeError) as e: result["error"] = str(e) print(json.dumps(result, indent=2)) return 1 @@ -164,11 +164,15 @@ def main() -> int: fs_after = int(after_out.strip()) if rc == 0 and after_out.strip().isdigit() else None result["fs_bytes_after"] = fs_after - ebs_grew = ec2.describe_volumes(VolumeIds=[args.volume_id])["Volumes"][0]["Size"] == result["new_size_gib"] + try: + ebs_grew = ec2.describe_volumes(VolumeIds=[args.volume_id])["Volumes"][0]["Size"] == result["new_size_gib"] + except (ClientError, BotoCoreError) as e: + _fail(operations["verify_size"], f"Could not verify final volume size: {e}") + ebs_grew = False fs_grew = fs_before is not None and fs_after is not None and fs_after > fs_before if ebs_grew and fs_grew: operations["verify_size"]["passed"] = True - else: + elif "error" not in operations["verify_size"]: _fail( operations["verify_size"], f"Size did not grow as expected (ebs_grew={ebs_grew}, fs_before={fs_before}, fs_after={fs_after})", diff --git a/isvctl/configs/providers/my-isv/scripts/storage/volume_resize.py b/isvctl/configs/providers/my-isv/scripts/storage/volume_resize.py index 65675f16..4504c649 100644 --- a/isvctl/configs/providers/my-isv/scripts/storage/volume_resize.py +++ b/isvctl/configs/providers/my-isv/scripts/storage/volume_resize.py @@ -60,6 +60,9 @@ def main() -> int: parser.add_argument("--grow-gib", type=int, default=5, help="GiB to add to the volume") args = parser.parse_args() + if args.grow_gib <= 0: + parser.error("--grow-gib must be positive") + operations: dict[str, dict[str, Any]] = { "modify_volume": {"passed": False}, "grow_partition": {"passed": False}, diff --git a/isvctl/tests/test_aws_storage_scripts.py b/isvctl/tests/test_aws_storage_scripts.py index 8fc24e7e..64385a94 100644 --- a/isvctl/tests/test_aws_storage_scripts.py +++ b/isvctl/tests/test_aws_storage_scripts.py @@ -143,6 +143,18 @@ def test_nvme_serial_and_by_id_path_drop_dash() -> None: ) +def test_describe_instance_raises_when_missing() -> None: + """An empty DescribeInstances response surfaces as RuntimeError.""" + module = _load_script("create_volume.py") + + class EmptyEc2: + def describe_instances(self, InstanceIds: list[str]) -> dict[str, Any]: + return {"Reservations": []} + + with pytest.raises(RuntimeError, match="Instance not found"): + module.ebs.describe_instance(EmptyEc2(), "i-gone") + + def test_detach_and_delete_volume_treats_missing_as_gone() -> None: """A volume that is already gone is a successful cleanup (returns None).""" module = _load_script("teardown_volume.py") @@ -219,6 +231,27 @@ def test_create_volume_guest_setup_failure_fails_step( assert payload["operations"]["format"]["passed"] is False +def test_create_volume_missing_instance_emits_json( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A missing fixture instance returns JSON instead of an IndexError.""" + module = _load_script("create_volume.py") + + class EmptyEc2: + def describe_instances(self, InstanceIds: list[str]) -> dict[str, Any]: + return {"Reservations": []} + + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: EmptyEc2()) + monkeypatch.setattr(sys, "argv", ["create_volume.py", "--instance-id", "i-gone", "--key-file", "/tmp/k.pem"]) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert payload["success"] is False + assert "Instance not found" in payload["error"] + + # --------------------------------------------------------------------------- # snapshot_lifecycle.py # --------------------------------------------------------------------------- @@ -404,6 +437,50 @@ def fake_ssh(host: str, user: str, key: str, script: str, **kwargs: Any) -> tupl assert payload["operations"]["resize_filesystem"]["passed"] is False +def test_volume_resize_final_size_api_error_fails_step( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A transient API error on the final size check fails verify_size cleanly.""" + module = _load_script("volume_resize.py") + + class FlakyEc2(FakeEc2): + def __init__(self) -> None: + super().__init__(volume_size=10) + self.describe_count = 0 + + def describe_volumes(self, VolumeIds: list[str]) -> dict[str, Any]: + self.describe_count += 1 + if self.describe_count >= 2: + raise _client_error("InternalError") + return super().describe_volumes(VolumeIds) + + monkeypatch.setattr(module.boto3, "client", lambda *a, **k: FlakyEc2()) + monkeypatch.setattr(module.ebs, "wait_for_modification_complete", lambda *a, **k: "completed") + monkeypatch.setattr(module, "wait_for_ssh", lambda *a, **k: True) + + sizes = iter(["10000000000", "15000000000"]) + + def fake_ssh(host: str, user: str, key: str, script: str, **kwargs: Any) -> tuple[int, str, str]: + if "df -B1" in script: + return (0, next(sizes), "") + return (0, "", "") + + monkeypatch.setattr(module, "ssh_run", fake_ssh) + monkeypatch.setattr( + sys, + "argv", + ["volume_resize.py", "--instance-id", "i-fixture", "--volume-id", "vol-fixture", "--key-file", "/tmp/k.pem"], + ) + + exit_code = module.main() + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert payload["success"] is False + assert payload["operations"]["verify_size"]["passed"] is False + assert "Could not verify final volume size" in payload["operations"]["verify_size"]["error"] + + # --------------------------------------------------------------------------- # volume_persistence.py # ---------------------------------------------------------------------------