diff --git a/Makefile b/Makefile index 78ad182..eb28f84 100644 --- a/Makefile +++ b/Makefile @@ -3,14 +3,24 @@ all: workloads MULTIHART ?= 0 PLATFORM ?= $(if $(filter 1,$(MULTIHART)),qemu,nemu) HARTS ?= 2 +VIRT_BUILD ?= 0 QEMU_DEFAULT_DTB ?= xiangshan-qemu-nemu-mem2g DTS_ISA_CONFIG ?= kunminghu-v3 DTS_DIR := build/generated-dts export DTS_ISA_CONFIG +ifneq ($(strip $(VIRTUALIZATION)),) +$(error VIRTUALIZATION was removed; build the Linux image first, then run 'make virt bin=') +endif + ifeq ($(filter $(PLATFORM),nemu qemu),) $(error PLATFORM must be either nemu or qemu) endif +ifeq ($(filter 1,$(VIRT_BUILD)),1) +ifneq ($(filter 1,$(MULTIHART)),) +$(error VIRT_BUILD=1 does not accept MULTIHART=1; use VIRT_GUEST_HARTS) +endif +endif ifeq ($(filter $(DTS_ISA_CONFIG),kunminghu-v2 kunminghu-v3),) $(error DTS_ISA_CONFIG must be either kunminghu-v2 or kunminghu-v3) endif @@ -22,9 +32,12 @@ endif LINUX_DEFAULT_DTB := $(if $(DEFAULT_DTB),$(DEFAULT_DTB),$(if $(filter 1,$(MULTIHART)),,$(if $(filter qemu,$(PLATFORM)),$(QEMU_DEFAULT_DTB),))) LINUX_FIRMWARE_FILENAME := $(if $(filter qemu,$(PLATFORM)),fw_payload.qemu.bin,fw_payload.bin) +SBI_BUILD_DIR := $(if $(filter 1,$(MULTIHART)),build/opensbi-multihart,build/opensbi) +SBI_BIN := $(SBI_BUILD_DIR)/build/platform/generic/firmware/fw_jump.bin LINUX_ROOTFS_BUILD_VARS_HASH := $(shell printf '%s\n' \ 'multihart=$(if $(filter 1,$(MULTIHART)),1,0)' \ - 'harts=$(if $(filter 1,$(MULTIHART)),$(HARTS),1)' | sha256sum | cut -d ' ' -f 1) + 'harts=$(if $(filter 1,$(MULTIHART)),$(HARTS),1)' \ + 'virt_build=$(VIRT_BUILD)' | sha256sum | cut -d ' ' -f 1) LINUX_FIRMWARE_BUILD_VARS_HASH := $(shell printf '%s\n' \ 'multihart=$(if $(filter 1,$(MULTIHART)),1,0)' \ 'harts=$(if $(filter 1,$(MULTIHART)),$(HARTS),1)' \ @@ -69,11 +82,121 @@ $(LINUX_IMAGE): $(TOOLCHAIN_WRAPPER) br2-external/configs/nemu_defconfig br2-ext $(MAKE) -C $(BUILDROOT_DIR) BR2_EXTERNAL=$(abspath br2-external) nemu_defconfig $(MAKE) -C $(BUILDROOT_DIR) BR2_EXTERNAL=$(abspath br2-external) +ifeq ($(VIRT_BUILD),1) +virt: virt-inner +else +.PHONY: virt +virt: + @test -n "$(bin)" || { echo 'Usage: make virt bin=' >&2; exit 2; } + @if test -d "$(bin)"; then \ + found=0; \ + for input in "$(abspath $(bin))"/*.bin; do \ + test -f "$$input" || continue; \ + found=1; \ + $(MAKE) --no-print-directory VIRT_BUILD=1 bin="$$input" virt-inner || exit $$?; \ + done; \ + test "$$found" -eq 1 || { echo "No .bin inputs found in $(bin)" >&2; exit 2; }; \ + elif test -f "$(bin)"; then \ + $(MAKE) --no-print-directory VIRT_BUILD=1 bin="$(abspath $(bin))" $(if $(elf),elf="$(abspath $(elf))") virt-inner; \ + else \ + echo "Input is not a regular file or directory: $(bin)" >&2; \ + exit 2; \ + fi +endif + +# Combine an existing image export with the per-case virtual packages without +# changing either source tree. +IMAGE_DIR ?= build/images/spec2006 +VIRT_DIR ?= build/virt +VIRT_IMAGE_DIR ?= $(IMAGE_DIR)-virt +.PHONY: virt-images +virt-images: + @python3 scripts/merge-virt-image-dir.py --image-dir "$(IMAGE_DIR)" --virt-dir "$(VIRT_DIR)" --output-dir "$(VIRT_IMAGE_DIR)" + +ifeq ($(VIRT_BUILD),1) +VIRT_GUEST_HARTS ?= 1 +VIRT_GUEST_MEMORY ?= 8G +VIRT_QEMU_START_TIMEOUT ?= 120 +VIRT_INNER_QEMU_VERSION := 11.0.0 +VIRT_HOST_BUILDROOT_DIR ?= build/buildroot-virt-host +VIRT_HOST_CONFIG := $(VIRT_HOST_BUILDROOT_DIR)/.config +VIRT_HOST_DEFCONFIG := br2-external/configs/nemu_virt_host_defconfig +VIRT_HOST_LINUX_CONFIG := br2-external/board/openxiangshan/nemu/linux-virt-host.config +ifeq ($(filter qemu,$(PLATFORM)),qemu) +VIRT_HOST_DTB ?= xiangshan-qemu-nemu-mem16g +VIRT_OUTER_QEMU_ARGS := --outer-qemu-machine nemu --outer-qemu-cpu rv64,h=true,sstc=false --outer-qemu-harts 1 --outer-qemu-memory 16G +else +VIRT_HOST_DTB ?= xiangshan-fpga-noAIA-mem16g-novec +VIRT_OUTER_QEMU_ARGS := +endif +VIRT_HOST_FIRMWARE := $(LINUX_FIRMWARE_FILENAME) +VIRT_HOST_MIN_MEMORY_BYTES ?= 17179869184 +VIRT_HOST_BUILD_STAMP := $(VIRT_HOST_BUILDROOT_DIR)/.build-complete.stamp +VIRT_HOST_IMAGE := $(VIRT_HOST_BUILDROOT_DIR)/images/Image +VIRT_HOST_ROOTFS := $(VIRT_HOST_BUILDROOT_DIR)/images/rootfs.cpio +VIRT_HOST_QEMU := $(VIRT_HOST_BUILDROOT_DIR)/target/usr/bin/qemu-system-riscv64 +VIRT_FAKEROOT := $(BUILDROOT_DIR)/output/host/bin/fakeroot +VIRT_GCPT_BUILD_DIR := build/LibCheckpointAlpha-virt +VIRT_GCPT_BIN := $(VIRT_GCPT_BUILD_DIR)/build/gcpt.bin +VIRT_GCPT_SOURCES := $(shell find bootloader/LibCheckpointAlpha -path '*/.git' -prune -o -type f -print 2>/dev/null) +VIRT_GCPT_DTS_SOURCES := dts/generate-nemu-board-dts.py dts/DTSGen.py dts/workload-builder-profiles.json +VIRT_GCPT_CONFIG_STAMP := build/LibCheckpointAlpha-virt-config/config.$(shell printf '%s\n' '$(VIRT_HOST_DTB)' | sha256sum | cut -d ' ' -f 1) +VIRT_BUILD_VARS_CONTENT := input=$(abspath $(bin))\nelf=$(if $(elf),$(abspath $(elf)))\nplatform=$(PLATFORM)\nguest_harts=$(VIRT_GUEST_HARTS)\nguest_memory=$(VIRT_GUEST_MEMORY)\nqemu_start_timeout=$(VIRT_QEMU_START_TIMEOUT)\ninner_qemu_version=$(VIRT_INNER_QEMU_VERSION)\nhost_dtb=$(VIRT_HOST_DTB)\nhost_min_memory=$(VIRT_HOST_MIN_MEMORY_BYTES) +VIRT_BUILD_VARS_HASH := $(shell printf '%b' '$(VIRT_BUILD_VARS_CONTENT)' | sha256sum | cut -d ' ' -f 1) + +$(VIRT_HOST_CONFIG): $(BUILDROOT_DIR)/Makefile $(VIRT_HOST_DEFCONFIG) $(VIRT_HOST_LINUX_CONFIG) + mkdir -p "$(VIRT_HOST_BUILDROOT_DIR)" + @if test -f "$@"; then $(MAKE) -C $(BUILDROOT_DIR) O=$(abspath $(VIRT_HOST_BUILDROOT_DIR)) BR2_EXTERNAL=$(abspath br2-external) linux-dirclean qemu-dirclean; fi + $(MAKE) -C $(BUILDROOT_DIR) O=$(abspath $(VIRT_HOST_BUILDROOT_DIR)) BR2_EXTERNAL=$(abspath br2-external) nemu_virt_host_defconfig + +$(VIRT_HOST_BUILD_STAMP): $(VIRT_HOST_CONFIG) $(VIRT_HOST_DEFCONFIG) $(VIRT_HOST_LINUX_CONFIG) + $(MAKE) -C $(BUILDROOT_DIR) O=$(abspath $(VIRT_HOST_BUILDROOT_DIR)) BR2_EXTERNAL=$(abspath br2-external) + test -s "$(VIRT_HOST_IMAGE)" + test -s "$(VIRT_HOST_ROOTFS)" + test -x "$(VIRT_HOST_QEMU)" + @test -n "$$(find "$(VIRT_HOST_BUILDROOT_DIR)/build" -maxdepth 1 -type d -name 'qemu-*' -print -quit)" + @host_kernel_config="$$(find "$(VIRT_HOST_BUILDROOT_DIR)/build" -path '*/linux-*/.config' -print -quit)"; test -n "$$host_kernel_config"; grep -qx 'CONFIG_VIRTUALIZATION=y' "$$host_kernel_config"; grep -qx 'CONFIG_KVM=y' "$$host_kernel_config" + touch "$@" + +$(VIRT_GCPT_CONFIG_STAMP): + mkdir -p "$(@D)" + rm -f "$(@D)"/config.* + touch "$@" + +$(VIRT_GCPT_BIN): scripts/build-gcpt.sh scripts/dts-config.sh $(TOOLCHAIN_WRAPPER) $(VIRT_GCPT_CONFIG_STAMP) $(VIRT_GCPT_SOURCES) $(VIRT_GCPT_DTS_SOURCES) + CROSS_COMPILE="$(abspath $(BUILDROOT_DIR)/output/host/bin)/riscv64-linux-" GCPT_IMPLEMENTATION=alpha DEFAULT_DTB="$(VIRT_HOST_DTB)" DTS_TEMPLATE_DIR="$(abspath $(DTS_DIR))" bash scripts/build-gcpt.sh bootloader/LibCheckpointAlpha $(VIRT_GCPT_BUILD_DIR) + +VIRT_INPUT_NAME := $(notdir $(bin)) +VIRT_INPUT_CASE := $(if $(filter fw_payload.bin fw_payload.qemu.bin,$(VIRT_INPUT_NAME)),$(notdir $(patsubst %/,%,$(dir $(bin)))),$(if $(filter %.fw_payload.qemu.bin,$(VIRT_INPUT_NAME)),$(patsubst %.fw_payload.qemu.bin,%,$(VIRT_INPUT_NAME)),$(if $(filter %.fw_payload.bin,$(VIRT_INPUT_NAME)),$(patsubst %.fw_payload.bin,%,$(VIRT_INPUT_NAME)),$(patsubst %.bin,%,$(VIRT_INPUT_NAME))))) +VIRT_INPUT_DIR := build/virt/$(VIRT_INPUT_CASE) +VIRT_INPUT_VARS := $(VIRT_INPUT_DIR)/vars.$(VIRT_BUILD_VARS_HASH).stamp +VIRT_INPUT_PACKAGE := $(VIRT_INPUT_DIR)/package.stamp +VIRT_INPUT_FIRMWARE := $(VIRT_INPUT_DIR)/host/$(VIRT_HOST_FIRMWARE) + +$(VIRT_INPUT_VARS): + @mkdir -p "$(@D)" + @rm -f "$(@D)"/vars.*.stamp + @printf '%b\n' '$(VIRT_BUILD_VARS_CONTENT)' > "$@" + +$(VIRT_INPUT_PACKAGE): $(bin) $(if $(elf),$(abspath $(elf))) $(VIRT_HOST_BUILD_STAMP) $(VIRT_INPUT_VARS) scripts/build-virtual-workload.py + @test -n "$(bin)" || { echo 'Usage: make virt bin=' >&2; exit 2; } + @mkdir -p "$(VIRT_INPUT_DIR)" + @$(VIRT_FAKEROOT) -- python3 scripts/build-virtual-workload.py input-package --input-bin "$(abspath $(bin))" $(if $(elf),--input-elf "$(abspath $(elf))") --host-rootfs "$(VIRT_HOST_ROOTFS)" --host-image "$(VIRT_HOST_IMAGE)" --guest-buildroot-output "$(BUILDROOT_DIR)/output" --host-buildroot-output "$(VIRT_HOST_BUILDROOT_DIR)" --out-dir "$(VIRT_INPUT_DIR)" --guest-harts "$(VIRT_GUEST_HARTS)" --guest-memory "$(VIRT_GUEST_MEMORY)" --qemu-start-timeout "$(VIRT_QEMU_START_TIMEOUT)" + @touch "$@" + +$(VIRT_INPUT_FIRMWARE): $(VIRT_INPUT_PACKAGE) $(VIRT_GCPT_BIN) $(SBI_BIN) $(VIRT_GCPT_DTS_SOURCES) scripts/build-firmware-linux.sh scripts/dts-config.sh + @CROSS_COMPILE="$(abspath $(BUILDROOT_DIR)/output/host/bin)/riscv64-linux-" DTC="$(abspath $(BUILDROOT_DIR)/output/host/bin/dtc)" DEFAULT_DTB="$(VIRT_HOST_DTB)" DTB_MIN_MEMORY_BYTES="$(VIRT_HOST_MIN_MEMORY_BYTES)" INITRAMFS_OFFSET_ALIGNMENT_MB=2 MULTIHART=0 HARTS=1 FIRMWARE_OUTPUT="$@" bash scripts/build-firmware-linux.sh "$(VIRT_GCPT_BIN)" "$(SBI_BUILD_DIR)" "$(DTS_DIR)" "$(VIRT_INPUT_DIR)/host/Image" "$(VIRT_INPUT_DIR)/host" + +$(VIRT_INPUT_DIR)/manifest.json: $(VIRT_INPUT_PACKAGE) $(VIRT_INPUT_FIRMWARE) scripts/build-virtual-workload.py + @$(VIRT_FAKEROOT) -- python3 scripts/build-virtual-workload.py manifest --out-dir "$(VIRT_INPUT_DIR)" --workload-name "$(VIRT_INPUT_CASE)" --platform "$(PLATFORM)" --guest-harts "$(VIRT_GUEST_HARTS)" --guest-memory "$(VIRT_GUEST_MEMORY)" --qemu-start-timeout "$(VIRT_QEMU_START_TIMEOUT)" --inner-qemu-version "$(VIRT_INNER_QEMU_VERSION)" --host-dtb "$(VIRT_HOST_DTB)" --host-min-memory-bytes "$(VIRT_HOST_MIN_MEMORY_BYTES)" $(VIRT_OUTER_QEMU_ARGS) --gcpt "$(VIRT_GCPT_BIN)" --opensbi "$(SBI_BIN)" --host-linux-revision "buildroot-selected" --opensbi-source bootloader/opensbi --opensbi-revision "$(shell git -C bootloader/opensbi rev-parse HEAD 2>/dev/null || printf unknown)" --host-defconfig "$(VIRT_HOST_DEFCONFIG)" --host-linux-config "$(VIRT_HOST_LINUX_CONFIG)" --guest-buildroot-output "$(BUILDROOT_DIR)/output" --host-buildroot-output "$(VIRT_HOST_BUILDROOT_DIR)" --buildroot-source "$(BUILDROOT_DIR)" + +virt-inner: $(VIRT_INPUT_DIR)/manifest.json + @printf '[virt] Output written to %s\n' "$(abspath $(VIRT_INPUT_DIR))" + +endif + # Build GCPT. Single-core firmware keeps LibCheckpointAlpha; LibCheckpoint is # used for the QEMU multi-hart checkpoint format. -SBI_BUILD_DIR := $(if $(filter 1,$(MULTIHART)),build/opensbi-multihart,build/opensbi) -SBI_BIN := $(SBI_BUILD_DIR)/build/platform/generic/firmware/fw_jump.bin - GCPT_IMPLEMENTATION := $(if $(filter 1,$(MULTIHART)),libcheckpoint,alpha) GCPT_SOURCE_DIR := $(if $(filter 1,$(MULTIHART)),bootloader/LibCheckpoint,bootloader/LibCheckpointAlpha) GCPT_BUILD_DIR := $(if $(filter 1,$(MULTIHART)),build/LibCheckpoint,build/LibCheckpointAlpha) @@ -191,9 +314,11 @@ AM_WORKLOAD_DIRS := $(patsubst workloads/am/%/build.sh,%,$(wildcard workloads/am LINUX_GENERIC_WORKLOADS := $(sort $(filter-out $(LINUX_WORKLOAD_RULE_DIRS),$(LINUX_WORKLOAD_DIRS))) AM_GENERIC_WORKLOADS := $(sort $(filter-out $(AM_WORKLOAD_RULE_DIRS),$(AM_WORKLOAD_DIRS))) # QEMU support covers the Linux workloads with QEMU-compatible firmware flows. +# Virtualization consumes a prebuilt input and does not alter workload registration. QEMU_SUPPORTED_LINUX_WORKLOADS := coremark spec2006 spec2017 -LINUX_ENABLED_GENERIC_WORKLOADS := $(if $(filter qemu,$(PLATFORM)),$(filter $(QEMU_SUPPORTED_LINUX_WORKLOADS),$(LINUX_GENERIC_WORKLOADS)),$(LINUX_GENERIC_WORKLOADS)) -LINUX_WORKLOAD_RULE_FILES := $(if $(filter qemu,$(PLATFORM)),$(wildcard $(foreach workload,$(QEMU_SUPPORTED_LINUX_WORKLOADS),workloads/linux/$(workload)/rules.mk)),$(wildcard workloads/linux/*/rules.mk)) +LINUX_QEMU_SUPPORTED_ONLY := $(filter qemu,$(PLATFORM)) +LINUX_ENABLED_GENERIC_WORKLOADS := $(if $(LINUX_QEMU_SUPPORTED_ONLY),$(filter $(QEMU_SUPPORTED_LINUX_WORKLOADS),$(LINUX_GENERIC_WORKLOADS)),$(LINUX_GENERIC_WORKLOADS)) +LINUX_WORKLOAD_RULE_FILES := $(if $(LINUX_QEMU_SUPPORTED_ONLY),$(wildcard $(foreach workload,$(QEMU_SUPPORTED_LINUX_WORKLOADS),workloads/linux/$(workload)/rules.mk)),$(wildcard workloads/linux/*/rules.mk)) $(foreach workload,$(LINUX_ENABLED_GENERIC_WORKLOADS),$(eval $(call add_workload_linux,$(workload)))) $(foreach workload,$(AM_GENERIC_WORKLOADS),$(eval $(call add_workload_am,$(workload)))) @@ -220,7 +345,7 @@ source: $(BUILDROOT_DIR)/Makefile make -C $(BUILDROOT_DIR) BR2_EXTERNAL=$(abspath br2-external) nemu_defconfig make -C $(BUILDROOT_DIR) BR2_EXTERNAL=$(abspath br2-external) source -# Build all all-in-one firmware images +# Build all all-in-one firmware images, or their virtual counterparts workloads: $(WORKLOADS_LINUX) $(WORKLOADS_AM_SENTINEL) # Build all rootfs diff --git a/README.md b/README.md index b2d9937..6db0b39 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,62 @@ You can also build a single workload with: - `make linux/workload_name` for a Linux workload. - `make am/workload_name` for an AM workload. +To package an already-built Linux firmware, or a bare-metal binary linked for +the standard QEMU `virt` machine, as a nested KVM Guest, pass its bin explicitly +to the independent `virt` target. A directory processes each direct child +`*.bin` input independently: + +```shell +make spec2006-images BENCH=bzip2 INPUT=chicken SPEC2006_ISO=/path/to/cpu2006.iso +make virt bin=build/images/spec2006/bin/bzip2_chicken.fw_payload.bin +make virt bin=build/images/spec2006/bin +``` + +To combine an existing image export and its matching virtual packages into a +new image directory, run: + +```shell +make virt-images \ + IMAGE_DIR=build/images/spec2006 \ + VIRT_DIR=build/virt \ + VIRT_IMAGE_DIR=build/images/spec2006-virt +``` + +The output keeps the original image layout. For each virtual case, `bin/` +contains the Host firmware, `kernel/`, `rootfs/`, and `dt/` contain its Host +components, while the workload ELF, command, configuration, and logs come from +the original image directory. Host GCPT and OpenSBI artifacts replace their +Guest counterparts. `manifest/.json` describes the reorganized files and +links to the full virtual manifest under `manifest/virt/`. Virtual packages +without a matching `bin/.fw_payload.bin` in `IMAGE_DIR` are ignored. +Neither source directory is modified. By default, the three paths above are +`build/images/spec2006`, `build/virt`, and `build/images/spec2006-virt` +respectively. `VIRT_IMAGE_DIR` must not already exist. + +For Linux firmware, `virt` scans the supplied image for its RISC-V Linux Image, +FDT and initramfs addresses; it does not assume a fixed 4 MiB guest-image +offset. A bare-metal `.bin` is booted directly as the Guest kernel and must +already target QEMU `virt`, write its terminal status as +`VIRT_WORKLOAD_EXIT=<0..255>`. If a matching ELF is not present in the sibling +`elf/` directory, `virt` creates a minimal ELF load wrapper at `0x80000000` and +records it in `build/virt//`. +The input bin/ELF are consumed as-is and are never rebuilt by `virt`. + +The images are exported with the usual SPEC CPU2006 image command: + +```shell +make spec2006-images BENCH=bzip2 INPUT=chicken \ + SPEC2006_ISO=/path/to/cpu2006.iso -jN +``` + +It writes the virtual Host firmware, device tree and manifest into +`build/virt//`. Inputs with the same case name share that output +directory. + +`PLATFORM=nemu` uses the FPGA host DTB (`xiangshan-fpga-noAIA-mem16g-novec` by +default); `PLATFORM=qemu` uses the QEMU `nemu` host DTB +(`xiangshan-qemu-nemu-mem16g` by default). Override either with `VIRT_HOST_DTB`. + Single-core workloads default to NEMU. QEMU support covers CoreMark, SPEC CPU2006, and SPEC CPU2017, which can be assembled with `PLATFORM=qemu`: diff --git a/br2-external/board/openxiangshan/nemu/linux-virt-host.config b/br2-external/board/openxiangshan/nemu/linux-virt-host.config new file mode 100644 index 0000000..1340f8c --- /dev/null +++ b/br2-external/board/openxiangshan/nemu/linux-virt-host.config @@ -0,0 +1,63 @@ +CONFIG_SYSVIPC=y +CONFIG_NO_HZ_IDLE=y +CONFIG_HIGH_RES_TIMERS=y +# CONFIG_BPF_SYSCALL is not set +# CONFIG_CGROUPS is not set +CONFIG_BLK_DEV_INITRD=y +CONFIG_INITRAMFS_SOURCE="" +# CONFIG_RD_GZIP is not set +# CONFIG_RD_BZIP2 is not set +# CONFIG_RD_LZMA is not set +# CONFIG_RD_XZ is not set +# CONFIG_RD_LZO is not set +# CONFIG_RD_LZ4 is not set +# CONFIG_RD_ZSTD is not set +# CONFIG_INITRAMFS_PRESERVE_MTIME is not set +CONFIG_NONPORTABLE=y +CONFIG_SMP=y +# CONFIG_RISCV_BOOT_SPINWAIT is not set +# CONFIG_COMPAT is not set +# CONFIG_EFI is not set +# CONFIG_SUSPEND is not set +CONFIG_VIRTUALIZATION=y +CONFIG_KVM=y +CONFIG_JUMP_LABEL=y +# CONFIG_GCC_PLUGINS is not set +CONFIG_NET=y +CONFIG_UNIX=y +# CONFIG_PACKET is not set +# CONFIG_INET is not set +# CONFIG_WIRELESS is not set +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +# CONFIG_NETDEVICES is not set +# CONFIG_LEGACY_PTYS is not set +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_8250_DW=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_SERIAL_EARLYCON_RISCV_SBI=y +CONFIG_SERIAL_UARTLITE=y +CONFIG_SERIAL_SIFIVE=y +CONFIG_SERIAL_SIFIVE_CONSOLE=y +CONFIG_HVC_RISCV_SBI=y +# CONFIG_HW_RANDOM is not set +# CONFIG_I2C is not set +# CONFIG_FB is not set +# CONFIG_HID_SUPPORT is not set +# CONFIG_USB_SUPPORT is not set +# CONFIG_VIRTIO_MENU is not set +# CONFIG_VHOST_MENU is not set +# CONFIG_IOMMU_SUPPORT is not set +# CONFIG_RPMSG is not set +# CONFIG_EXT4_FS is not set +# CONFIG_FAT_FS is not set +CONFIG_TMPFS=y +# CONFIG_HUGETLBFS is not set +# CONFIG_CONFIGFS_FS is not set +# CONFIG_MISC_FILESYSTEMS is not set +# CONFIG_NLS is not set +CONFIG_PRINTK_TIME=y +# CONFIG_RUNTIME_KERNEL_TESTING_MENU is not set +# CONFIG_RUNTIME_TESTING_MENU is not set + diff --git a/br2-external/configs/nemu_virt_host_defconfig b/br2-external/configs/nemu_virt_host_defconfig new file mode 100644 index 0000000..ab81a61 --- /dev/null +++ b/br2-external/configs/nemu_virt_host_defconfig @@ -0,0 +1,19 @@ +BR2_riscv=y +BR2_RISCV_ISA_RVC=y +BR2_TOOLCHAIN_EXTERNAL=y +BR2_TARGET_GENERIC_HOSTNAME="xiangshan-virt-host" +BR2_TARGET_GENERIC_ISSUE="Xiangshan NEMU virtualization host" +BR2_LINUX_KERNEL=y +BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y +BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE="${BR2_EXTERNAL}/board/openxiangshan/nemu/linux-virt-host.config" +BR2_TARGET_ROOTFS_CPIO=y +BR2_TARGET_ROOTFS_CPIO_FULL=y +BR2_TARGET_ROOTFS_CPIO_NONE=y +BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="${BR2_EXTERNAL}/board/openxiangshan/nemu/busybox-taskset.fragment" +BR2_PACKAGE_NEMU_TRAP=y +BR2_PACKAGE_QEMU=y +BR2_PACKAGE_QEMU_SYSTEM=y +BR2_PACKAGE_QEMU_CHOOSE_TARGETS=y +BR2_PACKAGE_QEMU_TARGET_RISCV64=y + + diff --git a/scripts/build-firmware-linux.sh b/scripts/build-firmware-linux.sh index 0e66150..6af2d47 100644 --- a/scripts/build-firmware-linux.sh +++ b/scripts/build-firmware-linux.sh @@ -13,6 +13,7 @@ DEFAULT_DTB="${DEFAULT_DTB:-}" DTB_MEMORY_PROFILE="${DTB_MEMORY_PROFILE:-}" DTB_MIN_MEMORY_BYTES="${DTB_MIN_MEMORY_BYTES:-}" DTB_REQUIRED_MIN_MEMORY_BYTES="${DTB_REQUIRED_MIN_MEMORY_BYTES:-}" +INITRAMFS_OFFSET_ALIGNMENT_MB="${INITRAMFS_OFFSET_ALIGNMENT_MB:-1}" HARTS="${HARTS:-2}" readonly MULTIHART_MAX_HARTS=128 readonly MULTIHART_KERNEL_OFFSET_MB=134 @@ -76,6 +77,18 @@ KERNEL_BEGIN_ADDR="$(dts_linux_kernel_address "$MEM_BEGIN" "$((KERNEL_MIN_OFFSET KERNEL_END_ADDR=$(( KERNEL_BEGIN_ADDR + KERNEL_SIZE )) KERNEL_OFFSET_KB=$(( (KERNEL_BEGIN_ADDR - MEM_BEGIN) / KILOBYTE )) INITRAMFS_BEGIN_ADDR=$(( (KERNEL_END_ADDR + MEGABYTE - 1) / MEGABYTE * MEGABYTE )) +case "$INITRAMFS_OFFSET_ALIGNMENT_MB" in + ''|*[!0-9]*) echo "INITRAMFS_OFFSET_ALIGNMENT_MB must be a positive integer" >&2; exit 1 ;; +esac +if [ "$INITRAMFS_OFFSET_ALIGNMENT_MB" -eq 0 ]; then + echo "INITRAMFS_OFFSET_ALIGNMENT_MB must be a positive integer" >&2 + exit 1 +fi +INITRAMFS_BEGIN_ADDR=$(( + (INITRAMFS_BEGIN_ADDR + INITRAMFS_OFFSET_ALIGNMENT_MB * MEGABYTE - 1) / + (INITRAMFS_OFFSET_ALIGNMENT_MB * MEGABYTE) * + (INITRAMFS_OFFSET_ALIGNMENT_MB * MEGABYTE) +)) INITRAMFS_OFFSET_KB=$(( (INITRAMFS_BEGIN_ADDR - MEM_BEGIN) / KILOBYTE )) INITRAMFS_SIZE=$(stat -c%s "$CPIO_ARCHIVE") INITRAMFS_END_ADDR=$(( INITRAMFS_BEGIN_ADDR + INITRAMFS_SIZE )) diff --git a/scripts/build-virtual-workload.py b/scripts/build-virtual-workload.py new file mode 100755 index 0000000..6d15e5a --- /dev/null +++ b/scripts/build-virtual-workload.py @@ -0,0 +1,1117 @@ +#!/usr/bin/env python3 +"""Build and describe Host/Guest pairs for virtual workload inputs.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shlex +import shutil +import stat +import struct +import subprocess +import tempfile +from pathlib import Path +from typing import Any + + +MEMORY_RE = re.compile(r"[1-9][0-9]*[KMGTkmgt]") +HARTS_RE = re.compile(r"[1-9][0-9]*") +NEMU_EXEC_RE = re.compile(r"^(?:/bin/)?nemu-exec(?:[ \t]+(.+))?$") +WORKLOAD_INIT_ACTIONS = {"sysinit", "wait", "once"} +HOST_FIRMWARE_NAMES = {"nemu": "fw_payload.bin", "qemu": "fw_payload.qemu.bin"} +HOST_ENTROPY_SEED = hashlib.sha256( + b"workload-builder virt host deterministic seed v1" +).digest() +FDT_MAGIC = 0xD00DFEED +FDT_BEGIN_NODE = 1 +FDT_END_NODE = 2 +FDT_PROP = 3 +FDT_NOP = 4 +FDT_END = 9 +RISCV_IMAGE_MAGIC = b"RSC\x05" + + +class InputKind: + LINUX = "linux" + BAREMETAL = "baremetal" + + +def read_be32(data: bytes, offset: int) -> int: + return int.from_bytes(data[offset : offset + 4], "big") + + +def read_be_cells(data: bytes, offset: int, cells: int) -> int: + end = offset + cells * 4 + if end > len(data): + raise ValueError("truncated FDT cell array") + value = 0 + for index in range(cells): + value = (value << 32) | read_be32(data, offset + index * 4) + return value + + +def fdt_strings(data: bytes, offset: int, size: int) -> bytes: + end = offset + size + if offset < 0 or end > len(data): + raise ValueError("truncated FDT strings block") + return data[offset:end] + + +def fdt_string(strings: bytes, offset: int) -> str: + if offset < 0 or offset >= len(strings): + raise ValueError("invalid FDT property name offset") + end = strings.find(b"\0", offset) + if end < 0: + raise ValueError("unterminated FDT property name") + return strings[offset:end].decode("ascii") + + +def parse_fdt(data: bytes, offset: int) -> dict[str, int] | None: + if offset < 0 or offset + 40 > len(data) or read_be32(data, offset) != FDT_MAGIC: + return None + totalsize = read_be32(data, offset + 4) + struct_offset = read_be32(data, offset + 8) + strings_offset = read_be32(data, offset + 12) + version = read_be32(data, offset + 20) + struct_size = read_be32(data, offset + 36) + strings_size = read_be32(data, offset + 32) + if version < 16 or totalsize < 40 or offset + totalsize > len(data): + return None + strings = fdt_strings(data, offset + strings_offset, strings_size) + struct_start = offset + struct_offset + struct_end = struct_start + struct_size + if struct_start < offset or struct_end > offset + totalsize: + return None + + stack: list[str] = [] + address_cells = 2 + size_cells = 2 + memory_base = None + memory_size = None + initrd_start = None + initrd_end = None + cursor = struct_start + try: + while cursor < struct_end: + token = read_be32(data, cursor) + cursor += 4 + if token == FDT_BEGIN_NODE: + end = data.find(b"\0", cursor, struct_end) + if end < 0: + return None + stack.append(data[cursor:end].decode("ascii")) + cursor = (end + 4) & ~3 + elif token == FDT_END_NODE: + if not stack: + return None + stack.pop() + elif token == FDT_PROP: + if cursor + 8 > struct_end: + return None + length = read_be32(data, cursor) + name_offset = read_be32(data, cursor + 4) + cursor += 8 + value_end = cursor + length + if value_end > struct_end: + return None + name = fdt_string(strings, name_offset) + path = "/" + "/".join(part for part in stack if part) + if path == "/" and name == "#address-cells" and length >= 4: + address_cells = read_be32(data, cursor) + elif path == "/" and name == "#size-cells" and length >= 4: + size_cells = read_be32(data, cursor) + elif path == "/chosen" and name == "linux,initrd-start": + initrd_start = read_be_cells(data, cursor, length // 4) + elif path == "/chosen" and name == "linux,initrd-end": + initrd_end = read_be_cells(data, cursor, length // 4) + elif stack and stack[-1].startswith("memory@") and name == "reg": + if length >= (address_cells + size_cells) * 4: + memory_base = read_be_cells(data, cursor, address_cells) + memory_size = read_be_cells( + data, cursor + address_cells * 4, size_cells + ) + cursor = (value_end + 3) & ~3 + elif token == FDT_NOP: + continue + elif token == FDT_END: + break + else: + return None + except (UnicodeDecodeError, ValueError): + return None + if None in (memory_base, memory_size, initrd_start, initrd_end): + return None + if initrd_end <= initrd_start or memory_size <= 0: + return None + return { + "dtb_offset": offset, + "dtb_size": totalsize, + "memory_base": int(memory_base), + "memory_size": int(memory_size), + "initrd_start": int(initrd_start), + "initrd_end": int(initrd_end), + } + + +def find_linux_layout(firmware: Path) -> dict[str, int] | None: + data = firmware.read_bytes() + images: list[tuple[int, int]] = [] + cursor = 0 + while True: + marker = data.find(RISCV_IMAGE_MAGIC, cursor) + if marker < 56: + break + image_offset = marker - 56 + if image_offset + 64 <= len(data): + text_offset = int.from_bytes(data[image_offset + 8 : image_offset + 16], "little") + image_size = int.from_bytes(data[image_offset + 16 : image_offset + 24], "little") + if image_size > 0 and image_offset + image_size <= len(data): + images.append((image_offset, image_size)) + cursor = marker + 4 + candidates: list[dict[str, int]] = [] + marker = FDT_MAGIC.to_bytes(4, "big") + offset = data.find(marker) + while offset >= 0: + fdt = parse_fdt(data, offset) + if fdt is not None: + initrd_start = fdt["initrd_start"] - fdt["memory_base"] + initrd_end = fdt["initrd_end"] - fdt["memory_base"] + if 0 <= initrd_start < initrd_end <= len(data): + for image_offset, image_size in images: + if image_offset < initrd_start <= len(data) and image_offset + image_size <= initrd_start: + candidates.append( + { + **fdt, + "image_offset": image_offset, + "image_size": image_size, + "initrd_offset": initrd_start, + "initrd_size": initrd_end - initrd_start, + } + ) + offset = data.find(marker, offset + 4) + if len(candidates) != 1: + return None + return candidates[0] + + +def resolve_input_elf(bin_path: Path, explicit: Path | None = None) -> Path | None: + case = bin_path.name + for suffix in (".fw_payload.qemu.bin", ".fw_payload.bin", ".bin"): + if case.endswith(suffix): + case = case[: -len(suffix)] + break + if case in {"fw_payload", "fw_payload.qemu"}: + case = bin_path.parent.name + candidates = [] + if explicit is not None: + candidates.append(explicit) + else: + candidates.extend( + ( + bin_path.parent / "elf" / f"{case}.elf", + bin_path.parent.parent / "elf" / f"{case}.elf", + ) + ) + elf = next((candidate for candidate in candidates if candidate.is_file() and candidate.stat().st_size), None) + if elf is None: + return None + header = subprocess.check_output(["readelf", "-h", str(elf)], text=True) + if "Machine:" not in header or "RISC-V" not in header: + raise ValueError(f"input ELF is not RISC-V: {elf}") + return elf + + +def classify_input( + bin_path: Path, elf: Path | None = None +) -> tuple[str, Path | None, dict[str, int] | None]: + require_file(bin_path) + layout = find_linux_layout(bin_path) + if layout is not None: + resolved_elf = resolve_input_elf(bin_path, elf) + if resolved_elf is None: + raise FileNotFoundError("matching ELF is required for Linux firmware") + return InputKind.LINUX, resolved_elf, layout + if bin_path.read_bytes().find(RISCV_IMAGE_MAGIC) >= 0: + raise ValueError( + f"Linux Image marker found but no unique DTB/initrd layout was found: {bin_path}" + ) + if bin_path.suffix == ".bin": + resolved_elf = resolve_input_elf(bin_path, elf) + return InputKind.BAREMETAL, resolved_elf, None + raise ValueError( + f"input is neither a recognized Linux firmware nor a bare-metal .bin: {bin_path}" + ) + + +def run(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[Any]: + return subprocess.run(command, check=True, **kwargs) + + +def make_raw_bin_elf(binary: Path, destination: Path) -> None: + data_size = binary.stat().st_size + file_offset = 0x1000 + ident = b"\x7fELF" + bytes((2, 1, 1, 0)) + bytes(8) + elf_header = struct.pack( + "<16sHHIQQQIHHHHHH", + ident, + 2, + 0xF3, + 1, + 0x80000000, + 64, + 0, + 0, + 64, + 56, + 1, + 0, + 0, + 0, + ) + program_header = struct.pack( + " None: + destination.mkdir(parents=True, exist_ok=True) + with archive.open("rb") as source: + run( + ["cpio", "-idmu", "--no-absolute-filenames"], + cwd=destination, + stdin=source, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + + +def pack_cpio(source: Path, archive: Path) -> None: + archive.parent.mkdir(parents=True, exist_ok=True) + temporary = archive.with_name(f".{archive.name}.tmp") + with temporary.open("wb") as output: + run( + [ + "sh", + "-c", + "find . -print0 | cpio --null -o -H newc --owner=0:0", + ], + cwd=source, + stdout=output, + stderr=subprocess.PIPE, + ) + os.replace(temporary, archive) + + +def validate_resources(guest_harts: str | int, guest_memory: str) -> tuple[int, str]: + harts = str(guest_harts) + if not HARTS_RE.fullmatch(harts): + raise ValueError("VIRT_GUEST_HARTS must be a positive integer") + if not MEMORY_RE.fullmatch(guest_memory): + raise ValueError( + "VIRT_GUEST_MEMORY must be a positive size such as 512M, 8G, or 1T" + ) + return int(harts), guest_memory.upper() + + +def validate_qemu_start_timeout(timeout: str | int) -> int: + value = str(timeout) + if not re.fullmatch(r"[1-9][0-9]*", value): + raise ValueError( + "VIRT_QEMU_START_TIMEOUT must be a positive integer number of seconds" + ) + return int(value) + + +def install_guest_runtime(root: Path, buildroot_output: Path) -> None: + for name in ("nemu-trap", "nemu-exec"): + destination = root / "bin" / name + copy_file(buildroot_output / "target" / "bin" / name, destination) + destination.chmod(destination.stat().st_mode | 0o111) + + +def install_guest_console(root: Path) -> None: + """Provide the early kernel console before devtmpfs is mounted.""" + console = root / "dev" / "console" + console.parent.mkdir(parents=True, exist_ok=True) + if os.path.lexists(console): + metadata = console.lstat() + if stat.S_ISCHR(metadata.st_mode) and metadata.st_rdev == os.makedev(5, 1): + return + if console.is_dir(): + raise ValueError(f"Guest console path is a directory: {console}") + console.unlink() + os.mknod(console, stat.S_IFCHR | 0o600, os.makedev(5, 1)) + + +def workload_commands(inittab: Path) -> list[str]: + commands: list[str] = [] + for line_number, line in enumerate( + inittab.read_text(encoding="utf-8").splitlines(), start=1 + ): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + fields = line.split(":", 3) + if len(fields) != 4: + raise ValueError(f"invalid Guest inittab line {line_number}: {line}") + action = fields[2] + if action not in WORKLOAD_INIT_ACTIONS: + raise ValueError( + f"unsupported Guest inittab action {action!r} on line {line_number}" + ) + command = fields[3].strip() + match = NEMU_EXEC_RE.fullmatch(command) + if match: + command = (match.group(1) or "").strip() + if not command: + raise ValueError(f"empty Guest command on inittab line {line_number}") + commands.append(command) + if not commands: + raise ValueError(f"Guest inittab has no workload actions: {inittab}") + return commands + + +def guest_command_script(commands: list[str]) -> str: + lines = ["#!/bin/sh", *commands] + lines.extend(["status=$?", 'exit "$status"', ""]) + return "\n".join(lines) + + +def guest_trap_shim() -> str: + return "\n".join( + [ + "#!/bin/sh", + "status=${1:-0}", + "run_dir=${VIRT_RUN_DIR:-/run}", + "case \"$status\" in", + " 256|257|258) exit 0 ;;", + " -*|'') status=1 ;;", + " *[!0-9]*) status=1 ;;", + "esac", + 'if [ "$status" -gt 255 ] 2>/dev/null; then status=1; fi', + 'mkdir -p "$run_dir"', + 'if [ ! -e "$run_dir/virt-nemu-trap.status" ]; then', + ' printf \'%s\\n\' "$status" > "$run_dir/virt-nemu-trap.status"', + "fi", + "exit 0", + "", + ] + ) + + +def guest_runner_script() -> str: + return "\n".join( + [ + "#!/bin/sh", + "set -u", + "run_dir=${VIRT_RUN_DIR:-/run}", + "command_file=${VIRT_COMMAND_FILE:-/etc/virt-workload-command}", + "notify=${VIRT_NEMU_NOTIFY:-/bin/nemu-notify}", + 'mkdir -p "$run_dir"', + 'rm -f "$run_dir/virt-nemu-trap.status"', + '"$notify" 256', + '"$notify" 257', + "set +e", + '/bin/sh "$command_file"', + "status=$?", + "set -e", + 'if [ -s "$run_dir/virt-nemu-trap.status" ]; then', + ' status=$(cat "$run_dir/virt-nemu-trap.status")', + "fi", + "case \"$status\" in ''|*[!0-9]*) status=1 ;; esac", + 'if [ "$status" -gt 255 ]; then status=1; fi', + 'if [ "$status" -eq 0 ]; then printf \'TEST DONE!\\n\'; fi', + 'printf \'VIRT_WORKLOAD_EXIT=%s\\n\' "$status"', + 'if [ "${VIRT_TEST_NO_POWEROFF:-0}" -eq 1 ]; then exit "$status"; fi', + "while :; do sleep 1; done", + "", + ] + ) + + +def install_guest_exit_adapter(root: Path) -> None: + inittab = root / "etc" / "inittab" + direct_init = root / "init" + if direct_init.is_file() and not direct_init.is_symlink(): + workload_init = root / "sbin" / "virt-workload-init" + os.replace(direct_init, workload_init) + commands = ["/sbin/virt-workload-init"] + else: + if not inittab.is_file(): + raise FileNotFoundError(inittab) + commands = workload_commands(inittab) + + command_file = root / "etc" / "virt-workload-command" + command_file.write_text(guest_command_script(commands), encoding="utf-8") + command_file.chmod(0o755) + + trap = root / "bin" / "nemu-trap" + notify = root / "bin" / "nemu-notify" + copy_file(trap, notify) + notify.chmod(notify.stat().st_mode | 0o111) + trap.write_text(guest_trap_shim(), encoding="utf-8") + trap.chmod(0o755) + + runner = root / "sbin" / "virt-guest-run" + runner.write_text(guest_runner_script(), encoding="utf-8") + runner.chmod(0o755) + inittab.write_text("console::sysinit:/sbin/virt-guest-run\n", encoding="utf-8") + if not direct_init.exists() and not direct_init.is_symlink(): + direct_init.symlink_to("sbin/init") + + +def assemble_guest_rootfs( + destination: Path, workload_rootfs: Path, buildroot_output: Path +) -> None: + target = buildroot_output / "target" + if not target.is_dir(): + raise FileNotFoundError(target) + shutil.copytree(target, destination, symlinks=True) + extract_cpio(workload_rootfs, destination) + install_guest_runtime(destination, buildroot_output) + install_guest_console(destination) + install_guest_exit_adapter(destination) + + +def qemu_argv( + guest_harts: int, guest_memory: str, baremetal: bool = False +) -> list[str]: + argv = [ + "/usr/bin/qemu-system-riscv64", + "-machine", + "virt,aia=none", + "-bios", + "none", + "-accel", + "kvm", + "-cpu", + "host", + "-smp", + str(guest_harts), + "-m", + guest_memory, + "-display", + "none", + "-monitor", + "none", + "-serial", + "stdio", + "-no-reboot", + "-kernel", + "/guest/Image", + ] + if not baremetal: + argv.extend( + [ + "-initrd", + "/guest/rootfs.cpio", + "-append", + "console=ttyS0 earlycon=sbi panic=1", + ] + ) + return argv + + +def host_init_script( + guest_harts: int, + guest_memory: str, + start_timeout: int = 120, + baremetal: bool = False, +) -> str: + qemu_arguments = " ".join( + shlex.quote(arg) + for arg in qemu_argv(guest_harts, guest_memory, baremetal)[1:] + ) + return "\n".join( + [ + "#!/bin/sh", + "set -u", + "run_dir=${VIRT_RUN_DIR:-/run}", + "kmsg_dev=${VIRT_KMSG:-/dev/kmsg}", + "qemu_bin=${VIRT_QEMU_BIN:-/usr/bin/qemu-system-riscv64}", + "nemu_trap=${VIRT_NEMU_TRAP:-/bin/nemu-trap}", + "mkdir -p \"$run_dir\"", + "kmsg() { printf '%s\\n' \"$1\" >> \"$kmsg_dev\"; }", + "kmsg 'VIRT_HOST: launching KVM guest'", + "rm -f \"$run_dir/virt-qemu.pipe\" \"$run_dir/virt-qemu.log\" \"$run_dir/virt-qemu.start-timeout\" \"$run_dir/virt-qemu.started\" \"$run_dir/virt-qemu.pid\" \"$run_dir/virt-qemu.completed\"", + ": > \"$run_dir/virt-qemu.log\"", + "mkfifo \"$run_dir/virt-qemu.pipe\"", + f"start_timeout=${{VIRT_QEMU_START_TIMEOUT:-{start_timeout}}}", + "cr=$(printf '\\r')", + f'"$qemu_bin" {qemu_arguments} "$run_dir/virt-qemu.pipe" 2>&1 &', + "qemu_pid=$!", + "(", + " marker_count=0", + " workload_status=", + " while IFS= read -r line || [ -n \"$line\" ]; do", + " line=${line%\"$cr\"}", + " printf '%s\\n' \"$line\" >> \"$run_dir/virt-qemu.log\"", + " printf '%s\\n' \"$line\" >> \"$kmsg_dev\"", + " printf '%s\\n' 1 > \"$run_dir/virt-qemu.started\"", + " case \"$line\" in", + " VIRT_WORKLOAD_EXIT=[0-9]|VIRT_WORKLOAD_EXIT=[0-9][0-9]|VIRT_WORKLOAD_EXIT=[0-9][0-9][0-9])", + " marker_count=$((marker_count + 1))", + " workload_status=${line#VIRT_WORKLOAD_EXIT=}", + " if [ \"$marker_count\" -eq 1 ]; then", + " printf '%s\\n' 1 > \"$run_dir/virt-qemu.completed\"", + " kill -9 \"$qemu_pid\" 2>/dev/null || true", + " fi", + " ;;", + " esac", + " done < \"$run_dir/virt-qemu.pipe\"", + " result=1", + " if [ -s \"$run_dir/virt-qemu.start-timeout\" ]; then", + " kmsg \"VIRT_HOST: QEMU/KVM produced no output within ${start_timeout}s\"", + " elif [ \"$marker_count\" -ne 1 ]; then", + " kmsg \"VIRT_HOST: expected one Guest exit marker, found $marker_count\"", + " elif [ -z \"$workload_status\" ] || [ \"$workload_status\" -gt 255 ] 2>/dev/null; then", + " kmsg 'VIRT_HOST: invalid Guest exit marker'", + " else", + " result=$workload_status", + " fi", + " kmsg \"VIRT_HOST: workload exit $result\"", + " exec \"$nemu_trap\" \"$result\"", + ") &", + "reader_pid=$!", + "printf '%s\\n' \"$qemu_pid\" > \"$run_dir/virt-qemu.pid\"", + "(", + " remaining=$start_timeout", + " while [ \"$remaining\" -gt 0 ] && [ ! -s \"$run_dir/virt-qemu.started\" ]; do", + " sleep 1", + " if [ -s \"$run_dir/virt-qemu.completed\" ] || ! kill -0 \"$qemu_pid\" 2>/dev/null; then exit 0; fi", + " remaining=$((remaining - 1))", + " done", + " if [ ! -s \"$run_dir/virt-qemu.started\" ] && kill -0 \"$qemu_pid\" 2>/dev/null; then", + " printf '%s\\n' 1 > \"$run_dir/virt-qemu.start-timeout\"", + " qemu_state=$(awk '/^State:/ { print $2 }' \"/proc/$qemu_pid/status\" 2>/dev/null || true)", + " qemu_wchan=$(cat \"/proc/$qemu_pid/wchan\" 2>/dev/null || true)", + " qemu_syscall=$(cat \"/proc/$qemu_pid/syscall\" 2>/dev/null || true)", + " kmsg \"VIRT_HOST: QEMU state=${qemu_state:-unknown} wchan=${qemu_wchan:-unknown} syscall=${qemu_syscall:-unknown}\"", + " kill \"$qemu_pid\" 2>/dev/null || true", + " sleep 1", + " kill -9 \"$qemu_pid\" 2>/dev/null || true", + " fi", + ") &", + "watchdog_pid=$!", + "wait \"$qemu_pid\" 2>/dev/null || true", + "kill \"$watchdog_pid\" 2>/dev/null || true", + "wait \"$watchdog_pid\" 2>/dev/null || true", + "wait \"$reader_pid\" 2>/dev/null", + "result=$?", + "rm -f \"$run_dir/virt-qemu.pipe\" \"$run_dir/virt-qemu.pid\"", + "if [ \"${VIRT_TEST_NO_POWEROFF:-0}\" -eq 1 ]; then exit \"$result\"; fi", + "while :; do :; done", + "", + ] + ) + + +def find_vmlinux(output_dir: Path) -> Path: + matches = sorted((output_dir / "build").glob("linux-*/vmlinux")) + if len(matches) != 1: + raise ValueError(f"expected one vmlinux under {output_dir}, found {matches}") + return matches[0] + + +def require_file(path: Path) -> None: + if not path.is_file() or path.stat().st_size == 0: + raise FileNotFoundError(path) + + +def copy_file(source: Path, destination: Path) -> None: + require_file(source) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(f".{destination.name}.tmp") + shutil.copy2(source, temporary) + os.replace(temporary, destination) + + +def package(args: argparse.Namespace) -> None: + guest_harts, guest_memory = validate_resources(args.guest_harts, args.guest_memory) + start_timeout = validate_qemu_start_timeout(args.qemu_start_timeout) + for path in ( + args.guest_rootfs, + args.host_rootfs, + args.guest_image, + args.host_image, + ): + require_file(path) + + output = args.out_dir + guest_dir = output / "guest" + host_dir = output / "host" + guest_dir.mkdir(parents=True, exist_ok=True) + host_dir.mkdir(parents=True, exist_ok=True) + + copy_file(args.guest_image, guest_dir / "Image") + copy_file(find_vmlinux(args.guest_buildroot_output), guest_dir / "vmlinux") + copy_file(args.host_image, host_dir / "Image") + copy_file(find_vmlinux(args.host_buildroot_output), host_dir / "vmlinux") + + with tempfile.TemporaryDirectory(prefix="virt-guest-") as temp: + guest_root = Path(temp) / "rootfs" + assemble_guest_rootfs( + guest_root, args.guest_rootfs, args.guest_buildroot_output + ) + pack_cpio(guest_root, guest_dir / "rootfs.cpio") + + with tempfile.TemporaryDirectory(prefix="virt-host-") as temp: + host_root = Path(temp) / "rootfs" + extract_cpio(args.host_rootfs, host_root) + qemu = host_root / "usr" / "bin" / "qemu-system-riscv64" + require_file(qemu) + seed = host_root / "var" / "lib" / "seedrng" / "seed.credit" + seed.parent.mkdir(parents=True, exist_ok=True) + seed.write_bytes(HOST_ENTROPY_SEED) + seed.chmod(0o400) + guest = host_root / "guest" + guest.mkdir(parents=True, exist_ok=True) + copy_file(guest_dir / "Image", guest / "Image") + copy_file(guest_dir / "rootfs.cpio", guest / "rootfs.cpio") + init_script = host_root / "etc" / "init.d" / "S99virt-workload" + init_script.parent.mkdir(parents=True, exist_ok=True) + init_script.write_text( + host_init_script(guest_harts, guest_memory, start_timeout), encoding="utf-8" + ) + init_script.chmod(0o755) + pack_cpio(host_root, host_dir / "rootfs.cpio") + + +def package_input(args: argparse.Namespace) -> None: + kind, workload_elf, layout = classify_input(args.input_bin, args.input_elf) + if kind == InputKind.BAREMETAL: + if workload_elf is None: + with tempfile.TemporaryDirectory(prefix="virt-raw-elf-") as temp: + workload_elf = Path(temp) / "workload.elf" + make_raw_bin_elf(args.input_bin, workload_elf) + package_baremetal(args, workload_elf) + copy_file(workload_elf, args.out_dir / "guest" / "workload.elf") + else: + package_baremetal(args, workload_elf) + copy_file(workload_elf, args.out_dir / "guest" / "workload.elf") + (args.out_dir / "input.json").write_text( + json.dumps( + { + "kind": kind, + "bin": str(args.input_bin), + "bin_sha256": sha256(args.input_bin), + "elf": str(args.out_dir / "guest" / "workload.elf"), + "elf_sha256": sha256(args.out_dir / "guest" / "workload.elf"), + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return + assert layout is not None + firmware_data = args.input_bin.read_bytes() + with tempfile.TemporaryDirectory(prefix="virt-input-") as temp: + temp_root = Path(temp) + guest_image = temp_root / "Image" + guest_rootfs = temp_root / "rootfs.cpio" + guest_image.write_bytes( + firmware_data[layout["image_offset"] : layout["image_offset"] + layout["image_size"]] + ) + guest_rootfs.write_bytes( + firmware_data[layout["initrd_offset"] : layout["initrd_offset"] + layout["initrd_size"]] + ) + package( + argparse.Namespace( + guest_rootfs=guest_rootfs, + host_rootfs=args.host_rootfs, + guest_image=guest_image, + host_image=args.host_image, + guest_buildroot_output=args.guest_buildroot_output, + host_buildroot_output=args.host_buildroot_output, + out_dir=args.out_dir, + guest_harts=args.guest_harts, + guest_memory=args.guest_memory, + qemu_start_timeout=args.qemu_start_timeout, + ) + ) + copy_file(workload_elf, args.out_dir / "guest" / "workload.elf") + (args.out_dir / "input.json").write_text( + json.dumps( + { + "kind": kind, + "bin": str(args.input_bin), + "elf": str(workload_elf), + "layout": layout, + "bin_sha256": sha256(args.input_bin), + "elf_sha256": sha256(workload_elf), + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + + +def package_baremetal(args: argparse.Namespace, guest_image: Path) -> None: + guest_harts, guest_memory = validate_resources(args.guest_harts, args.guest_memory) + start_timeout = validate_qemu_start_timeout(args.qemu_start_timeout) + for path in (guest_image, args.host_rootfs, args.host_image): + require_file(path) + + output = args.out_dir + guest_dir = output / "guest" + host_dir = output / "host" + guest_dir.mkdir(parents=True, exist_ok=True) + host_dir.mkdir(parents=True, exist_ok=True) + copy_file(guest_image, guest_dir / "Image") + copy_file(args.host_image, host_dir / "Image") + copy_file(find_vmlinux(args.host_buildroot_output), host_dir / "vmlinux") + + with tempfile.TemporaryDirectory(prefix="virt-host-") as temp: + host_root = Path(temp) / "rootfs" + extract_cpio(args.host_rootfs, host_root) + qemu = host_root / "usr" / "bin" / "qemu-system-riscv64" + require_file(qemu) + seed = host_root / "var" / "lib" / "seedrng" / "seed.credit" + seed.parent.mkdir(parents=True, exist_ok=True) + seed.write_bytes(HOST_ENTROPY_SEED) + seed.chmod(0o400) + guest = host_root / "guest" + guest.mkdir(parents=True, exist_ok=True) + copy_file(guest_dir / "Image", guest / "Image") + init_script = host_root / "etc" / "init.d" / "S99virt-workload" + init_script.parent.mkdir(parents=True, exist_ok=True) + init_script.write_text( + host_init_script(guest_harts, guest_memory, start_timeout, baremetal=True), + encoding="utf-8", + ) + init_script.chmod(0o755) + pack_cpio(host_root, host_dir / "rootfs.cpio") + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def component(path: Path, relative: str) -> dict[str, str | int]: + require_file(path) + return {"path": relative, "bytes": path.stat().st_size, "sha256": sha256(path)} + + +def parse_needed(elf: Path) -> list[str]: + output = subprocess.check_output(["readelf", "-d", str(elf)], text=True) + return re.findall(r"Shared library: \[([^]]+)]", output) + + +def validate_qemu_root(root: Path) -> dict[str, Any]: + qemu = root / "usr" / "bin" / "qemu-system-riscv64" + require_file(qemu) + header = subprocess.check_output(["readelf", "-h", str(qemu)], text=True) + if "Machine:" not in header or "RISC-V" not in header: + raise ValueError(f"Host QEMU is not a RISC-V ELF: {qemu}") + program_headers = subprocess.check_output(["readelf", "-l", str(qemu)], text=True) + if "Requesting program interpreter" not in program_headers: + raise ValueError(f"Host QEMU is not dynamically linked: {qemu}") + + roots = [qemu] + seen: set[str] = set() + resolved: dict[str, Path] = {} + while roots: + current = roots.pop() + for name in parse_needed(current): + if name in seen: + continue + seen.add(name) + candidates = [ + path + for prefix in (root / "lib", root / "usr" / "lib") + for path in prefix.glob(f"**/{name}") + ] + if not candidates: + raise ValueError(f"missing dynamic dependency {name} for {current}") + dependency = candidates[0] + resolved[name] = dependency + roots.append(dependency) + return { + "sha256": sha256(qemu), + "needed": sorted(resolved), + "library_hashes": {name: sha256(path) for name, path in sorted(resolved.items())}, + } + + +def directory_version(output_dir: Path, prefix: str) -> str: + matches = sorted(path.name.removeprefix(prefix) for path in (output_dir / "build").glob(f"{prefix}*")) + if len(matches) != 1: + raise ValueError(f"expected one {prefix} version under {output_dir}, found {matches}") + return matches[0] + + +def config_record(path: Path) -> dict[str, str]: + require_file(path) + return {"path": str(path), "sha256": sha256(path)} + + +def source_record(path: Path, revision: str) -> dict[str, str | int]: + if path.is_dir(): + record: dict[str, str | int] = {"path": str(path)} + else: + record = component(path, str(path)) + record["revision"] = revision + return record + + +def buildroot_config(output_dir: Path) -> Path: + """Resolve both Buildroot O= and the repository's in-tree output layout.""" + candidates = (output_dir / ".config", output_dir.parent / ".config") + for candidate in candidates: + if candidate.is_file(): + return candidate + raise ValueError(f"Buildroot configuration not found near {output_dir}") + + +def source_record_from_buildroot(output_dir: Path, revision: str) -> dict[str, str | int]: + matches = sorted((output_dir / "build").glob("linux-*/Makefile")) + if len(matches) != 1: + raise ValueError(f"expected one Linux source under {output_dir}, found {matches}") + return {"path": str(matches[0].parent), "revision": revision} + + +def buildroot_version(source: Path) -> str: + makefile = (source / "Makefile").read_text(encoding="utf-8") + match = re.search(r"^(?:export\s+)?BR2_VERSION\s*:?=\s*(.+)$", makefile, re.MULTILINE) + return match.group(1).strip() if match else "unknown" + + +def write_manifest(args: argparse.Namespace) -> None: + guest_harts, guest_memory = validate_resources(args.guest_harts, args.guest_memory) + start_timeout = validate_qemu_start_timeout(args.qemu_start_timeout) + output = args.out_dir + guest = output / "guest" + host = output / "host" + input_metadata = output / "input.json" + input_data = ( + json.loads(input_metadata.read_text(encoding="utf-8")) + if input_metadata.is_file() + else None + ) + baremetal = input_data is not None and input_data.get("kind") == InputKind.BAREMETAL + dtb = host / "dt" / f"{args.host_dtb}.dtb" + firmware_name = HOST_FIRMWARE_NAMES[args.platform] + host_kernel_config = find_vmlinux(args.host_buildroot_output).parent / ".config" + guest_kernel_config = None + guest_buildroot_config = None + if not baremetal: + guest_kernel_config = find_vmlinux(args.guest_buildroot_output).parent / ".config" + guest_buildroot_config = buildroot_config(args.guest_buildroot_output) + host_buildroot_config = buildroot_config(args.host_buildroot_output) + opensbi_root = args.opensbi.parents[4] + opensbi_config = opensbi_root / "platform" / "generic" / "configs" / "defconfig" + + with tempfile.TemporaryDirectory(prefix="virt-manifest-") as temp: + host_root = Path(temp) / "rootfs" + extract_cpio(host / "rootfs.cpio", host_root) + qemu = validate_qemu_root(host_root) + + qemu_version = directory_version(args.host_buildroot_output, "qemu-") + if qemu_version != args.inner_qemu_version: + raise ValueError( + f"inner QEMU version mismatch: expected {args.inner_qemu_version}, found {qemu_version}" + ) + + argv = qemu_argv(guest_harts, guest_memory, baremetal) + guest_data: dict[str, Any] = { + "harts": guest_harts, + "memory": guest_memory, + "Image": component(guest / "Image", "guest/Image"), + } + if baremetal: + guest_data["kind"] = InputKind.BAREMETAL + guest_data["workload_elf"] = component(guest / "workload.elf", "guest/workload.elf") + else: + assert guest_kernel_config is not None and guest_buildroot_config is not None + guest_data.update( + { + "vmlinux": component(guest / "vmlinux", "guest/vmlinux"), + "rootfs": component(guest / "rootfs.cpio", "guest/rootfs.cpio"), + "buildroot_config": config_record(guest_buildroot_config), + "kernel_config": config_record(guest_kernel_config), + } + ) + data = { + "schema_version": 2, + "workload": args.workload_name, + "platform": args.platform, + "guest": guest_data, + "host": { + "dtb_name": args.host_dtb, + "minimum_memory_bytes": int(args.host_min_memory_bytes), + "Image": component(host / "Image", "host/Image"), + "vmlinux": component(host / "vmlinux", "host/vmlinux"), + "rootfs": component(host / "rootfs.cpio", "host/rootfs.cpio"), + "firmware": component( + host / firmware_name, f"host/{firmware_name}" + ), + "dtb": component(dtb, f"host/dt/{args.host_dtb}.dtb"), + "gcpt": component(args.gcpt, str(args.gcpt)), + "opensbi": component(args.opensbi, str(args.opensbi)), + "opensbi_config": config_record(opensbi_config), + "defconfig": config_record(args.host_defconfig), + "linux_config": config_record(args.host_linux_config), + "buildroot_config": config_record(host_buildroot_config), + "kernel_config": config_record(host_kernel_config), + "linux_source": source_record_from_buildroot( + args.host_buildroot_output, args.host_linux_revision + ), + "opensbi_source": source_record( + args.opensbi_source, args.opensbi_revision + ), + }, + "qemu": { + "binary": "/usr/bin/qemu-system-riscv64", + "version": qemu_version, + "dtb_mode": "qemu-generated", + "argv": argv, + "startup_timeout_seconds": start_timeout, + **qemu, + }, + "buildroot": { + "version": buildroot_version(args.buildroot_source), + "source": str(args.buildroot_source), + }, + } + workload_elf = guest / "workload.elf" + if workload_elf.is_file(): + data["guest"]["workload_elf"] = component(workload_elf, "guest/workload.elf") + input_metadata = output / "input.json" + if input_metadata.is_file(): + data["input"] = json.loads(input_metadata.read_text(encoding="utf-8")) + if args.platform == "qemu": + outer_qemu_harts, outer_qemu_memory = validate_resources( + args.outer_qemu_harts, args.outer_qemu_memory + ) + data["outer_qemu"] = { + "machine": args.outer_qemu_machine, + "cpu": args.outer_qemu_cpu, + "harts": outer_qemu_harts, + "memory": outer_qemu_memory, + "firmware": component( + host / firmware_name, f"host/{firmware_name}" + ), + "dtb_name": args.host_dtb, + "dtb": component(dtb, f"host/dt/{args.host_dtb}.dtb"), + "argv": [ + "qemu-system-riscv64", + "-machine", + args.outer_qemu_machine, + "-accel", + "tcg", + "-cpu", + args.outer_qemu_cpu, + "-smp", + str(outer_qemu_harts), + "-m", + outer_qemu_memory, + "-nographic", + "-no-reboot", + "-bios", + f"host/{firmware_name}", + ], + } + manifest = output / "manifest.json" + temporary = manifest.with_name(".manifest.json.tmp") + temporary.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(temporary, manifest) + + +def add_resource_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--guest-harts", default="1") + parser.add_argument("--guest-memory", default="8G") + parser.add_argument("--qemu-start-timeout", default="120") + + +def add_package_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--host-rootfs", type=Path, required=True) + parser.add_argument("--host-image", type=Path, required=True) + parser.add_argument("--guest-buildroot-output", type=Path, required=True) + parser.add_argument("--host-buildroot-output", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + add_resource_args(parser) + + +def main() -> int: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="action", required=True) + + package_parser = subparsers.add_parser("package") + package_parser.add_argument("--guest-rootfs", type=Path, required=True) + package_parser.add_argument("--host-rootfs", type=Path, required=True) + package_parser.add_argument("--guest-image", type=Path, required=True) + package_parser.add_argument("--host-image", type=Path, required=True) + package_parser.add_argument("--guest-buildroot-output", type=Path, required=True) + package_parser.add_argument("--host-buildroot-output", type=Path, required=True) + package_parser.add_argument("--out-dir", type=Path, required=True) + add_resource_args(package_parser) + + input_parser = subparsers.add_parser("input-package") + input_parser.add_argument("--input-bin", type=Path, required=True) + input_parser.add_argument("--input-elf", type=Path) + add_package_args(input_parser) + + manifest_parser = subparsers.add_parser("manifest") + manifest_parser.add_argument("--out-dir", type=Path, required=True) + manifest_parser.add_argument("--workload-name", required=True) + manifest_parser.add_argument("--platform", choices=("nemu", "qemu"), required=True) + manifest_parser.add_argument("--host-dtb", required=True) + manifest_parser.add_argument("--host-min-memory-bytes", required=True) + manifest_parser.add_argument("--outer-qemu-machine") + manifest_parser.add_argument("--outer-qemu-cpu") + manifest_parser.add_argument("--outer-qemu-harts") + manifest_parser.add_argument("--outer-qemu-memory") + manifest_parser.add_argument("--gcpt", type=Path, required=True) + manifest_parser.add_argument("--opensbi", type=Path, required=True) + manifest_parser.add_argument("--host-linux-revision", required=True) + manifest_parser.add_argument("--opensbi-source", type=Path, required=True) + manifest_parser.add_argument("--opensbi-revision", required=True) + manifest_parser.add_argument("--host-defconfig", type=Path, required=True) + manifest_parser.add_argument("--host-linux-config", type=Path, required=True) + manifest_parser.add_argument("--guest-buildroot-output", type=Path, required=True) + manifest_parser.add_argument("--host-buildroot-output", type=Path, required=True) + manifest_parser.add_argument("--buildroot-source", type=Path, required=True) + manifest_parser.add_argument("--inner-qemu-version", required=True) + add_resource_args(manifest_parser) + + args = parser.parse_args() + try: + if args.action == "package": + package(args) + elif args.action == "input-package": + package_input(args) + else: + write_manifest(args) + except (FileNotFoundError, ValueError, subprocess.CalledProcessError) as exc: + print(f"error: {exc}", file=os.sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/export-linux-debug-artifacts.sh b/scripts/export-linux-debug-artifacts.sh index ba6bd20..070030f 100755 --- a/scripts/export-linux-debug-artifacts.sh +++ b/scripts/export-linux-debug-artifacts.sh @@ -16,9 +16,13 @@ GCPT_ELF="$(realpath "${GCPT_ELF:?GCPT_ELF must be set}")" GCPT_BIN="$(realpath "${GCPT_BIN:?GCPT_BIN must be set}")" source "$(dirname "${BASH_SOURCE[0]}")/dts-config.sh" -mapfile -t vmlinux_files < <(find "$BUILDROOT_DIR/output/build" -path '*/vmlinux' -type f -print | sort) +BUILDROOT_OUTPUT_DIR="$BUILDROOT_DIR/output" +if [ ! -d "$BUILDROOT_OUTPUT_DIR/build" ]; then + BUILDROOT_OUTPUT_DIR="$BUILDROOT_DIR" +fi +mapfile -t vmlinux_files < <(find "$BUILDROOT_OUTPUT_DIR/build" -path '*/vmlinux' -type f -print | sort) if [ "${#vmlinux_files[@]}" -ne 1 ]; then - echo "Expected exactly one vmlinux under $BUILDROOT_DIR/output/build, found ${#vmlinux_files[@]}" >&2 + echo "Expected exactly one vmlinux under $BUILDROOT_OUTPUT_DIR/build, found ${#vmlinux_files[@]}" >&2 exit 1 fi diff --git a/scripts/merge-virt-image-dir.py b/scripts/merge-virt-image-dir.py new file mode 100644 index 0000000..de691c9 --- /dev/null +++ b/scripts/merge-virt-image-dir.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Combine a regular image export with per-case virt packages.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import sys +from pathlib import Path + + +def is_within(path: Path, parent: Path) -> bool: + try: + path.relative_to(parent) + except ValueError: + return False + return True + + +def artifact_name(manifest: dict[str, object]) -> str: + workload = str(manifest["workload"]) + input_data = manifest.get("input") + input_name = "" + if isinstance(input_data, dict) and input_data.get("bin"): + input_name = Path(str(input_data["bin"])).name + if input_name not in {"fw_payload.bin", "fw_payload.qemu.bin"}: + for suffix in (".fw_payload.qemu.bin", ".fw_payload.bin", ".bin"): + if input_name.endswith(suffix): + return input_name[: -len(suffix)] + if Path(workload).name != workload or workload in {"", ".", ".."}: + raise ValueError(f"invalid workload name in virtual manifest: {workload!r}") + return workload + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def merge(image_dir: Path, virt_dir: Path, output_dir: Path) -> None: + image_dir = image_dir.resolve() + virt_dir = virt_dir.resolve() + output_dir = output_dir.resolve() + for source in (image_dir, virt_dir): + if not source.is_dir(): + raise ValueError(f"source directory does not exist: {source}") + if output_dir.exists() or output_dir.is_symlink(): + raise ValueError(f"output directory already exists: {output_dir}") + if is_within(output_dir, image_dir) or is_within(output_dir, virt_dir): + raise ValueError("output directory must not be inside a source directory") + + virtual_cases = [] + names = set() + for case_dir in sorted(virt_dir.iterdir(), key=lambda item: item.name): + if not case_dir.is_dir() or not (case_dir / "manifest.json").is_file(): + continue + manifest = json.loads((case_dir / "manifest.json").read_text(encoding="utf-8")) + name = artifact_name(manifest) + if not (image_dir / "bin" / f"{name}.fw_payload.bin").is_file(): + continue + if name in names: + raise ValueError(f"duplicate virtual workload name: {name}") + names.add(name) + virtual_cases.append((case_dir, manifest, name)) + if not virtual_cases: + raise ValueError(f"no virtual workloads in {virt_dir} match {image_dir}") + + first_manifest = virtual_cases[0][1] + gcpt_hash = first_manifest["host"]["gcpt"]["sha256"] + opensbi_hash = first_manifest["host"]["opensbi"]["sha256"] + for _, manifest, name in virtual_cases[1:]: + if manifest["host"]["gcpt"]["sha256"] != gcpt_hash: + raise ValueError(f"virtual workload uses a different Host GCPT: {name}") + if manifest["host"]["opensbi"]["sha256"] != opensbi_hash: + raise ValueError(f"virtual workload uses a different Host OpenSBI: {name}") + + gcpt_bin = Path(first_manifest["host"]["gcpt"]["path"]) + gcpt_elf = gcpt_bin.with_name("gcpt") + opensbi_bin = Path(first_manifest["host"]["opensbi"]["path"]) + opensbi_elf = opensbi_bin.with_name("fw_jump.elf") + opensbi_config_data = first_manifest["host"].get("opensbi_config") + if isinstance(opensbi_config_data, dict): + opensbi_config = Path(str(opensbi_config_data["path"])) + else: + opensbi_config = opensbi_bin.parents[4] / "platform" / "generic" / "configs" / "defconfig" + + output_dir.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(image_dir, output_dir) + shared_files = { + gcpt_bin: output_dir / "gcpt" / "gcpt.bin", + gcpt_elf: output_dir / "gcpt" / "gcpt.elf", + opensbi_elf: output_dir / "opensbi" / "fw_jump.elf", + opensbi_config: output_dir / "opensbi" / "defconfig", + } + for source, destination in shared_files.items(): + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + + for case_dir, manifest, name in virtual_cases: + firmware = manifest["host"]["firmware"] + dtb = manifest["host"]["dtb"] + kernel_config = Path(manifest["host"]["kernel_config"]["path"]) + system_map = kernel_config.parent / "System.map" + paths = { + "firmware": f"bin/{name}.fw_payload.bin", + "Image": f"kernel/{name}.Image", + "vmlinux": f"kernel/{name}.vmlinux", + "kernel_config": f"kernel/{name}.config", + "system_map": f"kernel/{name}.System.map", + "rootfs": f"rootfs/{name}.rootfs.cpio", + "dtb": f"dt/{name}.dtb", + "dts": f"dt/{name}.dts", + } + for relative in paths.values(): + (output_dir / relative).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(case_dir / firmware["path"], output_dir / paths["firmware"]) + shutil.copy2(case_dir / "host" / "Image", output_dir / paths["Image"]) + shutil.copy2(case_dir / "host" / "vmlinux", output_dir / paths["vmlinux"]) + shutil.copy2(kernel_config, output_dir / paths["kernel_config"]) + shutil.copy2(system_map, output_dir / paths["system_map"]) + shutil.copy2(case_dir / "host" / "rootfs.cpio", output_dir / paths["rootfs"]) + shutil.copy2(case_dir / dtb["path"], output_dir / paths["dtb"]) + dts = case_dir / "host" / "dt" / f"{manifest['host']['dtb_name']}.dts" + shutil.copy2(dts, output_dir / paths["dts"]) + manifest_path = output_dir / "manifest" / f"{name}.json" + virtual_manifest_path = output_dir / "manifest" / "virt" / f"{name}.json" + manifest_path.parent.mkdir(parents=True, exist_ok=True) + virtual_manifest_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(case_dir / "manifest.json", virtual_manifest_path) + image_manifest = { + "case": name, + "virtual_manifest": f"manifest/virt/{name}.json", + "firmware": { + "file": paths["firmware"], + "sha256": firmware["sha256"], + }, + "kernel": { + "Image": paths["Image"], + "vmlinux": paths["vmlinux"], + "system_map": paths["system_map"], + "config": paths["kernel_config"], + "sha256": sha256(output_dir / paths["vmlinux"]), + }, + "initramfs": { + "file": paths["rootfs"], + "sha256": manifest["host"]["rootfs"]["sha256"], + }, + "device_tree": { + "dtb": paths["dtb"], + "dts": paths["dts"], + "basename": manifest["host"]["dtb_name"], + "sha256": dtb["sha256"], + }, + "gcpt": { + "elf": "gcpt/gcpt.elf", + "binary": "gcpt/gcpt.bin", + "elf_sha256": sha256(gcpt_elf), + "binary_sha256": gcpt_hash, + }, + "opensbi": { + "elf": "opensbi/fw_jump.elf", + "config": "opensbi/defconfig", + "sha256": sha256(opensbi_elf), + }, + } + manifest_path.write_text( + json.dumps(image_manifest, indent=2) + "\n", encoding="utf-8" + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Copy an image export and virt packages into one image directory." + ) + parser.add_argument("--image-dir", type=Path, required=True) + parser.add_argument("--virt-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + try: + merge(args.image_dir, args.virt_dir, args.output_dir) + except (OSError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + print(f"[virt-images] Output written to {args.output_dir.resolve()}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/workloads/linux/coremark-pro/README.md b/workloads/linux/coremark-pro/README.md index 7f1378c..ad752da 100644 --- a/workloads/linux/coremark-pro/README.md +++ b/workloads/linux/coremark-pro/README.md @@ -14,3 +14,15 @@ The workload downloads and builds the CoreMark-PRO benchmark suite from source. 2. Each benchmark is run with parameters `-c1 -w1 -v1` (single thread verification run). 3. If any benchmark fails, the system halts with the error code from that benchmark. 4. System is halted with nemu-trap after all benchmarks complete. + +## Virtualization + +Build CoreMark-PRO as a nested KVM guest and run the Host firmware with QEMU: + +```sh +make linux/coremark-pro PLATFORM=qemu -jN +make virt bin=build/linux-workloads/coremark-pro/fw_payload.qemu.bin +QEMU_BIN=/path/to/qemu-system-riscv64 QEMU_MEMORY=16G \ + bash scripts/run-qemu.sh \ + build/virt/coremark-pro/host/fw_payload.qemu.bin +``` diff --git a/workloads/linux/coremark/README.md b/workloads/linux/coremark/README.md index 57c45db..48f63fe 100644 --- a/workloads/linux/coremark/README.md +++ b/workloads/linux/coremark/README.md @@ -57,3 +57,15 @@ QEMU_BIN=/path/to/qemu-system-riscv64 QEMU_MEMORY=8G \ HARTS=2 bash scripts/run-qemu.sh \ build/linux-workloads/coremark/fw_payload.qemu.bin ``` + +## Virtualization + +Build CoreMark as a nested KVM guest and run the Host firmware with QEMU: + +```bash +make linux/coremark PLATFORM=qemu -jN +make virt bin=build/linux-workloads/coremark/fw_payload.qemu.bin +QEMU_BIN=/path/to/qemu-system-riscv64 QEMU_MEMORY=16G \ + bash scripts/run-qemu.sh \ + build/virt/coremark/host/fw_payload.qemu.bin +``` diff --git a/workloads/linux/gapbs/README.md b/workloads/linux/gapbs/README.md index 2685213..aa2c78c 100644 --- a/workloads/linux/gapbs/README.md +++ b/workloads/linux/gapbs/README.md @@ -45,6 +45,18 @@ make gapbs-images -jN # build all 18 images Images are written to `build/images/gapbs/bin/.fw_payload.bin`. +## Virtualization + +Build one GAPBS case as a nested KVM guest and run the Host firmware with QEMU: + +```sh +make linux/gapbs-bfs_road PLATFORM=qemu -jN +make virt bin=build/linux-workloads/gapbs/bfs_road/fw_payload.qemu.bin +QEMU_BIN=/path/to/qemu-system-riscv64 QEMU_MEMORY=16G \ + bash scripts/run-qemu.sh \ + build/virt/bfs_road/host/fw_payload.qemu.bin +``` + ## Graphs Kernels load a pre-serialized graph with `-f`, so graph construction is not diff --git a/workloads/linux/geekbench5/README.md b/workloads/linux/geekbench5/README.md index 1881ba1..edffe12 100644 --- a/workloads/linux/geekbench5/README.md +++ b/workloads/linux/geekbench5/README.md @@ -66,3 +66,15 @@ Geekbench Pro, so this workload builds the runnable CPU benchmark image. The workload always enables the takeover; there is no network-enabled mode in the generated image. + +## Virtualization + +Build Geekbench 5 as a nested KVM guest and run the Host firmware with QEMU: + +```sh +make linux/geekbench5 PLATFORM=qemu -jN +make virt bin=build/linux-workloads/geekbench5/fw_payload.qemu.bin +QEMU_BIN=/path/to/qemu-system-riscv64 QEMU_MEMORY=16G \ + bash scripts/run-qemu.sh \ + build/virt/geekbench5/host/fw_payload.qemu.bin +``` diff --git a/workloads/linux/geekbench6/README.md b/workloads/linux/geekbench6/README.md index 6fd8f59..9242b7d 100644 --- a/workloads/linux/geekbench6/README.md +++ b/workloads/linux/geekbench6/README.md @@ -70,3 +70,15 @@ Geekbench Pro, so this workload builds the runnable CPU benchmark image. The workload always enables the takeover; there is no network-enabled mode in the generated image. + +## Virtualization + +Build Geekbench 6 as a nested KVM guest and run the Host firmware with QEMU: + +```sh +make linux/geekbench6 PLATFORM=qemu -jN +make virt bin=build/linux-workloads/geekbench6/fw_payload.qemu.bin +QEMU_BIN=/path/to/qemu-system-riscv64 QEMU_MEMORY=16G \ + bash scripts/run-qemu.sh \ + build/virt/geekbench6/host/fw_payload.qemu.bin +``` diff --git a/workloads/linux/hello/README.md b/workloads/linux/hello/README.md index 358c75b..84e1f1e 100644 --- a/workloads/linux/hello/README.md +++ b/workloads/linux/hello/README.md @@ -16,3 +16,15 @@ The main reason of this it that a workload is needed to test `riscv64-nutshell_d - It is possible to use the same kernel on RISC-V devices with or without hardware floating point. The kernel itself will decide which ISA extensions to use, based on information provided by the device tree. So the solution for a Linux smoke test on nutshell is an assembly program installed as `/init`. + +## Virtualization + +Build Hello as a nested KVM guest and run the Host firmware with QEMU: + +```sh +make linux/hello PLATFORM=qemu -jN +make virt bin=build/linux-workloads/hello/fw_payload.qemu.bin +QEMU_BIN=/path/to/qemu-system-riscv64 QEMU_MEMORY=16G \ + bash scripts/run-qemu.sh \ + build/virt/hello/host/fw_payload.qemu.bin +``` diff --git a/workloads/linux/hello/source/hello.s b/workloads/linux/hello/source/hello.s index f79ae5b..0c9a895 100644 --- a/workloads/linux/hello/source/hello.s +++ b/workloads/linux/hello/source/hello.s @@ -20,7 +20,7 @@ _start: bne a0, t0, fail li a0, 0 # Exit status = 0 - .word 0x0005006b # NEMU trap instruction + .word 0x0005006b # NEMU trap instruction with rs1 = a0 fail: li a0, 1 diff --git a/workloads/linux/kvmtool/README.md b/workloads/linux/kvmtool/README.md index 4904daf..e861af2 100644 --- a/workloads/linux/kvmtool/README.md +++ b/workloads/linux/kvmtool/README.md @@ -14,3 +14,15 @@ This workload builds and runs `lkvm`, a lightweight virtual machine manager (kvm ## Note NEMU profiling mode is not turned on in this workload as it can cause problems when running virtualization. Simulating more than one core using `kvmtool` will also fail. + +## Virtualization + +Build kvmtool as a nested KVM guest and run the Host firmware with QEMU: + +```sh +make linux/kvmtool PLATFORM=qemu -jN +make virt bin=build/linux-workloads/kvmtool/fw_payload.qemu.bin +QEMU_BIN=/path/to/qemu-system-riscv64 QEMU_MEMORY=16G \ + bash scripts/run-qemu.sh \ + build/virt/kvmtool/host/fw_payload.qemu.bin +``` diff --git a/workloads/linux/litmus-tests-riscv/README.md b/workloads/linux/litmus-tests-riscv/README.md index 30c02e6..48f6f28 100644 --- a/workloads/linux/litmus-tests-riscv/README.md +++ b/workloads/linux/litmus-tests-riscv/README.md @@ -5,3 +5,15 @@ These are litmus tests for the RISC-V concurrency architecture, as used by members of the RISC-V Memory Model Task Group during the architecture development. This is a Linux workload. + +## Virtualization + +Build the litmus tests as a nested KVM guest and run the Host firmware with QEMU: + +```sh +make linux/litmus-tests-riscv PLATFORM=qemu -jN +make virt bin=build/linux-workloads/litmus-tests-riscv/fw_payload.qemu.bin +QEMU_BIN=/path/to/qemu-system-riscv64 QEMU_MEMORY=16G \ + bash scripts/run-qemu.sh \ + build/virt/litmus-tests-riscv/host/fw_payload.qemu.bin +``` diff --git a/workloads/linux/rocksdb/README.md b/workloads/linux/rocksdb/README.md index 3224102..a99e1e6 100644 --- a/workloads/linux/rocksdb/README.md +++ b/workloads/linux/rocksdb/README.md @@ -113,3 +113,16 @@ load/store share, followed by `mixgraph`. `randomtransaction` has the largest RVV share and vector-instruction count. A literal 100,000 yi (`100000` hundred-million) instructions is `1e13`, not `1e12`; use ten times the listed operation count for that target. + +## Virtualization + +Build one RocksDB case as a nested KVM guest and run the Host firmware with QEMU: + +```sh +make ROCKSDB_CROSS_COMPILE=/path/to/riscv64-unknown-linux-gnu- \ + linux/rocksdb-readwhilewriting PLATFORM=qemu -jN +make virt bin=build/linux-workloads/rocksdb/readwhilewriting/fw_payload.qemu.bin +QEMU_BIN=/path/to/qemu-system-riscv64 QEMU_MEMORY=16G \ + bash scripts/run-qemu.sh \ + build/virt/readwhilewriting/host/fw_payload.qemu.bin +``` diff --git a/workloads/linux/rvv-bench/README.md b/workloads/linux/rvv-bench/README.md index 479bd02..06a283d 100644 --- a/workloads/linux/rvv-bench/README.md +++ b/workloads/linux/rvv-bench/README.md @@ -13,3 +13,15 @@ The workload builds multiple vector-optimized benchmarks from source. It uses an 1. Executes the run.sh script which runs all benchmark executables sequentially 2. Each benchmark is executed and checked for successful completion 3. System is halted with nemu-trap after all benchmarks complete + +## Virtualization + +Build RVV-Bench as a nested KVM guest and run the Host firmware with QEMU: + +```sh +make linux/rvv-bench PLATFORM=qemu -jN +make virt bin=build/linux-workloads/rvv-bench/fw_payload.qemu.bin +QEMU_BIN=/path/to/qemu-system-riscv64 QEMU_MEMORY=16G \ + bash scripts/run-qemu.sh \ + build/virt/rvv-bench/host/fw_payload.qemu.bin +``` diff --git a/workloads/linux/spec2006/README.md b/workloads/linux/spec2006/README.md index b682f5f..9562dd2 100644 --- a/workloads/linux/spec2006/README.md +++ b/workloads/linux/spec2006/README.md @@ -299,3 +299,16 @@ SPEC2006 builds keep console output concise. Detailed logs are written to: redirected into the case-local `runspec-output` directory via `output_root`. - The source cfg in this repository is never passed to `runspec` directly; `runspec` is allowed to rewrite only the generated local cfg copy. + +## Virtualization + +Build one SPEC CPU2006 case as a nested KVM guest and run the Host firmware with QEMU: + +```sh +make linux/spec2006 BENCH=astar INPUT=biglakes \ + SPEC2006_ISO=/path/to/cpu2006.iso PLATFORM=qemu -jN +make virt bin=build/linux-workloads/spec2006/astar_biglakes/fw_payload.qemu.bin +QEMU_BIN=/path/to/qemu-system-riscv64 QEMU_MEMORY=16G \ + bash scripts/run-qemu.sh \ + build/virt/astar_biglakes/host/fw_payload.qemu.bin +``` diff --git a/workloads/linux/spec2006/rules.mk b/workloads/linux/spec2006/rules.mk index ff5ce69..f34f455 100644 --- a/workloads/linux/spec2006/rules.mk +++ b/workloads/linux/spec2006/rules.mk @@ -227,7 +227,7 @@ spec2006-images: spec2006-check-spec-iso echo "No SPEC2006 cases selected by SPEC2006_INPUT=$(SPEC2006_INPUT)"; \ exit 1; \ fi; \ - rm -rf "$(SPEC2006_IMAGE_DIR)/bin" "$(SPEC2006_IMAGE_DIR)/kernel" "$(SPEC2006_IMAGE_DIR)/rootfs" "$(SPEC2006_IMAGE_DIR)/elf" "$(SPEC2006_IMAGE_DIR)/cmd" "$(SPEC2006_IMAGE_DIR)/cfg" "$(SPEC2006_IMAGE_DIR)/gcpt" "$(SPEC2006_IMAGE_DIR)/dt" "$(SPEC2006_IMAGE_DIR)/manifest" "$(SPEC2006_IMAGE_DIR)/logs" "$(SPEC2006_IMAGE_DIR)/stamps"; \ + rm -rf "$(SPEC2006_IMAGE_DIR)/bin" "$(SPEC2006_IMAGE_DIR)/kernel" "$(SPEC2006_IMAGE_DIR)/opensbi" "$(SPEC2006_IMAGE_DIR)/rootfs" "$(SPEC2006_IMAGE_DIR)/elf" "$(SPEC2006_IMAGE_DIR)/cmd" "$(SPEC2006_IMAGE_DIR)/cfg" "$(SPEC2006_IMAGE_DIR)/gcpt" "$(SPEC2006_IMAGE_DIR)/dt" "$(SPEC2006_IMAGE_DIR)/manifest" "$(SPEC2006_IMAGE_DIR)/logs" "$(SPEC2006_IMAGE_DIR)/stamps"; \ total="$(words $(SPEC2006_IMAGE_CASES))"; \ i=0; \ for case in $(SPEC2006_IMAGE_CASES); do \ diff --git a/workloads/linux/spec2017/README.md b/workloads/linux/spec2017/README.md index 504a25e..783fba1 100644 --- a/workloads/linux/spec2017/README.md +++ b/workloads/linux/spec2017/README.md @@ -245,3 +245,16 @@ mode-specific defaults with `SPEC2017_RATE_CFG` and `SPEC2017_SPEED_CFG`. it does not depend on benchmark cfg selection. Installation staging uses a temporary local filesystem; override `SPEC2017_PREPARE_TMPDIR` if `/tmp` is not suitable. `xorriso` is required for this step. + +## Virtualization + +Build one SPEC CPU2017 case as a nested KVM guest and run the Host firmware with QEMU: + +```sh +make linux/spec2017 BENCH=mcf MODE=rate INPUT=ref \ + SPEC2017_ISO=/path/to/cpu2017.iso PLATFORM=qemu -jN +make virt bin=build/linux-workloads/spec2017/mcf_rate_refrate/fw_payload.qemu.bin +QEMU_BIN=/path/to/qemu-system-riscv64 QEMU_MEMORY=16G \ + bash scripts/run-qemu.sh \ + build/virt/mcf_rate_refrate/host/fw_payload.qemu.bin +``` diff --git a/workloads/linux/spec2026/README.md b/workloads/linux/spec2026/README.md index 337fc77..1a517df 100644 --- a/workloads/linux/spec2026/README.md +++ b/workloads/linux/spec2026/README.md @@ -141,3 +141,16 @@ build/generated-dts/xiangshan-fpga-noAIA-mem64g-novec.dts.in Re-running `make spec2026-images` rebuilds the export tree so the directory layout and contents stay complete and consistent. + +## Virtualization + +Build one SPEC CPU2026 case as a nested KVM guest and run the Host firmware with QEMU: + +```sh +make linux/spec2026 BENCH=706.stockfish_r \ + SPEC2026_ISO=/path/to/cpu2026-1.0.1.iso PLATFORM=qemu -jN +make virt bin=build/linux-workloads/spec2026/706.stockfish_r/fw_payload.qemu.bin +QEMU_BIN=/path/to/qemu-system-riscv64 QEMU_MEMORY=16G \ + bash scripts/run-qemu.sh \ + build/virt/706.stockfish_r/host/fw_payload.qemu.bin +``` diff --git a/workloads/linux/specjbb2015/README.md b/workloads/linux/specjbb2015/README.md index 7c006b4..926e0ff 100644 --- a/workloads/linux/specjbb2015/README.md +++ b/workloads/linux/specjbb2015/README.md @@ -19,3 +19,19 @@ The output is `build/linux-workloads/specjbb2015/fw_payload.bin`. The image uses one multithreaded JVM shared by all guest harts and the standard multi-hart checkpoint memory layout. The selected DTB must match `HARTS` and provide enough memory for the configured Java heap. + +## Virtualization + +Build SPECjbb2015 as a nested KVM guest and run the Host firmware with QEMU: + +```sh +make linux/specjbb2015 \ + SPECJBB_INPUT=/path/to/SPECjbb2015-1.03.iso \ + SPECJBB_RV_JDK_INPUT=/path/to/jdk25 \ + SPECJBB_MODE=COMPOSITE SPECJBB_JVM_XMS=4g SPECJBB_JVM_XMX=4g \ + PLATFORM=qemu -jN +make virt bin=build/linux-workloads/specjbb2015/fw_payload.qemu.bin VIRT_GUEST_HARTS=2 +QEMU_BIN=/path/to/qemu-system-riscv64 QEMU_MEMORY=16G \ + bash scripts/run-qemu.sh \ + build/virt/specjbb2015/host/fw_payload.qemu.bin +``` diff --git a/workloads/linux/stress-ng/README.md b/workloads/linux/stress-ng/README.md index e34827e..cd78356 100644 --- a/workloads/linux/stress-ng/README.md +++ b/workloads/linux/stress-ng/README.md @@ -67,3 +67,15 @@ To embed another supported device tree, pass its template basename: ```sh make linux/stress-ng DEFAULT_DTB=xiangshan-fpga-noAIA-novec -jN ``` + +## Virtualization + +Build stress-ng as a nested KVM guest and run the Host firmware with QEMU: + +```sh +make linux/stress-ng PLATFORM=qemu -jN +make virt bin=build/linux-workloads/stress-ng/fw_payload.qemu.bin +QEMU_BIN=/path/to/qemu-system-riscv64 QEMU_MEMORY=16G \ + bash scripts/run-qemu.sh \ + build/virt/stress-ng/host/fw_payload.qemu.bin +```