diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6e0d5e2244..ea9a42b331 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -355,6 +355,69 @@ jobs: if [ -n "$VERBS_FEATURES" ]; then features=(--features "$VERBS_FEATURES"); fi timeout --signal=TERM --kill-after=10s 300s cargo +1.96.0 test --locked --manifest-path cmd/racer-dataplane/Cargo.toml -p rdma-verbs --no-default-features "${features[@]}" + racer-gpu: + name: Racer GPU (${{ matrix.configuration }}) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - configuration: no-default-features + features: "" + native: false + - configuration: simulation + features: simulation + native: false + - configuration: native + features: native + native: true + - configuration: native-and-simulation + features: native,simulation + native: true + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/bin/racer-gpu-cargo + CARGO_BUILD_JOBS: "2" + GPU_FEATURES: ${{ matrix.features }} + LD_LIBRARY_PATH: ${{ github.workspace }}/bin/racer-gpu-native + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up pinned Rust + run: timeout --signal=TERM --kill-after=10s 300s rustup toolchain install 1.96.0 --profile minimal --component rustfmt,clippy + + # No CUDA toolkit on hosted runners: the shim builds without decrypt + # kernels and loads without a driver. Kernel math is covered by the + # C-core tests below and the kernel emulation step. + - name: Compile CUDA shim without kernels (not hardware validation) + if: matrix.native + run: timeout --signal=TERM --kill-after=10s 300s sh images/racer-dataplane/build-native-cuda.sh cmd/racer-dataplane/gpu/native bin/racer-gpu-native/libracer_gpu_cuda.so.1 + + - name: Require CUDA shim ABI and symbols (no hardware) + if: matrix.native + run: timeout --signal=TERM --kill-after=10s 300s cargo +1.96.0 test --locked --manifest-path cmd/racer-dataplane/Cargo.toml -p racer-gpu --no-default-features --features "$GPU_FEATURES" --lib ffi::tests::required_native_shim_loads_abi_and_symbols -- --exact --ignored + + - name: Run decrypt kernels on the CPU (kernel emulation) + if: matrix.native + run: timeout --signal=TERM --kill-after=10s 300s sh cmd/racer-dataplane/gpu/native/run-kernel-emulation.sh bin/racer-gpu-emulation + + - name: Check GPU formatting + run: timeout --signal=TERM --kill-after=10s 300s cargo +1.96.0 fmt --manifest-path cmd/racer-dataplane/Cargo.toml -p racer-gpu --check + + - name: Strict GPU Clippy + run: | + features=() + if [ -n "$GPU_FEATURES" ]; then features=(--features "$GPU_FEATURES"); fi + timeout --signal=TERM --kill-after=10s 300s cargo +1.96.0 clippy --locked --manifest-path cmd/racer-dataplane/Cargo.toml -p racer-gpu --all-targets --no-default-features "${features[@]}" -- -D warnings + + # Includes the C AES-GCM/CRC core check against racer-crypto (16 MiB pages). + - name: Test GPU crate without requiring GPU hardware + run: | + features=() + if [ -n "$GPU_FEATURES" ]; then features=(--features "$GPU_FEATURES"); fi + timeout --signal=TERM --kill-after=10s 300s cargo +1.96.0 test --locked --manifest-path cmd/racer-dataplane/Cargo.toml -p racer-gpu --no-default-features "${features[@]}" + runtime-miri: name: Runtime Miri (${{ matrix.group }}) runs-on: ubuntu-24.04 @@ -392,6 +455,17 @@ jobs: - name: Provision envtest and run controller API-server race tests run: timeout --signal=TERM --kill-after=10s 300s make racer-envtest-ci + racer-python: + name: Racer Python Client + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Run Python client and vLLM plugin tests + run: timeout --signal=TERM --kill-after=10s 540s make racer-python-test + # ---------- Orca Integration Tests ---------- # Spins up Garage and Azurite via testcontainers-go and runs the # orca in-process integration suite (internal/orca/inttest). Docker diff --git a/Makefile b/Makefile index 91b0b53a74..8cfade985e 100644 --- a/Makefile +++ b/Makefile @@ -581,6 +581,15 @@ racer-server-test: ## Lint and race-test the Racer server racer-test: racer-server-test ## Check the Racer controller +RACER_PY_VENV ?= $(CURDIR)/tmp/racer-python-venv + +.PHONY: racer-python-test +racer-python-test: ## Run the Racer Python client and vLLM plugin tests (no GPU needed) + @test -x "$(RACER_PY_VENV)/bin/pytest" || { timeout --signal=TERM --kill-after=10s 120s python3 -m venv "$(RACER_PY_VENV)" && \ + timeout --signal=TERM --kill-after=10s 300s "$(RACER_PY_VENV)/bin/pip" install -q pytest; } + cd clients/python/racer && PYTHONPATH=src timeout --signal=TERM --kill-after=10s 300s "$(RACER_PY_VENV)/bin/pytest" -q -p no:cacheprovider + cd clients/python/racer-vllm && PYTHONPATH=src:../racer/src timeout --signal=TERM --kill-after=10s 300s "$(RACER_PY_VENV)/bin/pytest" -q -p no:cacheprovider + $(SETUP_ENVTEST): @mkdir -p bin tmp/envtest-tools TMPDIR="$(CURDIR)/tmp/envtest-tools" GOBIN="$(CURDIR)/bin" timeout --signal=TERM --kill-after=10s 300s $(GOCMD) install sigs.k8s.io/controller-runtime/tools/setup-envtest@$(SETUP_ENVTEST_VERSION) diff --git a/NOTICE b/NOTICE index d6464eafd8..45e368030f 100644 --- a/NOTICE +++ b/NOTICE @@ -48,6 +48,15 @@ notices: link: https://docs.rs/crate/base64/0.22.1/source/LICENSE-APACHE - name: MIT License link: https://docs.rs/crate/base64/0.22.1/source/LICENSE-MIT + - dependency: cc + ecosystem: cargo + copyright: + - Copyright (c) 2014 Alex Crichton + license: + - name: Apache License, Version 2.0 + link: https://docs.rs/crate/cc/1.5.1/source/LICENSE-APACHE + - name: MIT License + link: https://docs.rs/crate/cc/1.5.1/source/LICENSE-MIT - dependency: chacha20poly1305 ecosystem: cargo copyright: diff --git a/clients/python/.gitignore b/clients/python/.gitignore new file mode 100644 index 0000000000..c4af65667e --- /dev/null +++ b/clients/python/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.egg-info/ +.pytest_cache/ +build/ +dist/ diff --git a/clients/python/racer-vllm/README.md b/clients/python/racer-vllm/README.md new file mode 100644 index 0000000000..501919aea9 --- /dev/null +++ b/clients/python/racer-vllm/README.md @@ -0,0 +1,83 @@ +# racer-vllm + +A vLLM plugin that loads model weights from a Racer cache. It adds the load +format `racer`. It uses the `racer-client` package. + +## Install + +``` +pip install ./clients/python/racer ./clients/python/racer-vllm +``` + +vLLM finds the plugin through the `vllm.general_plugins` entry point. It was +written for vLLM v0.25.x. + +## Model location + +A model lives at `racer:////`. The plugin reads +objects through `/run/racer//client/socket`. Set `RACER_SOCKET` to use +another socket. Object keys follow racer-object, so `//` +names the S3 object. + +## Use + +vLLM reads the model config and tokenizer before it loads weights, so these +small files must be on local disk first. Fetch them, for example in an init +container: + +``` +racer-vllm-fetch racer://models/my-bucket/llama-3-8b /models/llama-3-8b +``` + +This copies `config.json`, tokenizer files, `model.safetensors.index.json` +and similar files, plus small files named in the index metadata. It also +writes `racer-model.json`, which tells the loader where the weights are. + +Then start vLLM on that directory: + +``` +vllm serve /models/llama-3-8b --load-format racer \ + --model-loader-extra-config '{"staging_bytes": 268435456, "concurrency": 4}' +``` + +Extra config keys, all optional: + +| Key | Default | Meaning | +|---|---|---| +| `uri` | from `racer-model.json` | The `racer://` model location. | +| `staging_bytes` | 256 MiB | Size of each GPU staging buffer. | +| `concurrency` | 4 | Staging buffers, and reads in flight. | +| `clone` | true | Copy each tensor out of staging before handing it to vLLM. | +| `socket` | none | Client socket path. | +| `use_vmm` | true | Allow exportable `cuMemCreate` staging, needed for `gpu-read`. | + +On fast local disks, more reads in flight help only when the dataplane also +fetches further ahead. Set `concurrency` to 8 together with a larger +`RACER_RANGE_WINDOW_PAGES` and `RACER_GPU_STAGING_BYTES` on the dataplane. See +the GPU reads section of the Racer reference. + +## How it loads + +1. Find the safetensors files from `model.safetensors.index.json`, or use + `model.safetensors`. +2. For each file, read the 8-byte header length and the JSON header. +3. Group tensors into ranges that fit a staging buffer. Each range ends on a + Racer page boundary (16 MiB), so no page is read twice. A tensor that + crosses the boundary is split across two ranges and joined again before vLLM + sees it. Read each range into + GPU memory with `gpu-read`. If the dataplane cannot do that, use + `host-read` into pinned memory and copy, else the normal stream. +4. Give each tensor (right dtype and shape) to `model.load_weights`. While + vLLM uses one buffer, the next range is read into another. + +A tensor bigger than a staging buffer gets its own GPU allocation. + +With `clone` set to false, tensors are views into staging memory. They stay +valid only until vLLM asks for the next tensors after the buffer's last one. +Only use this when the model copies each weight at once. + +## Tests + +``` +make racer-python-test +``` diff --git a/clients/python/racer-vllm/pyproject.toml b/clients/python/racer-vllm/pyproject.toml new file mode 100644 index 0000000000..55540c18d5 --- /dev/null +++ b/clients/python/racer-vllm/pyproject.toml @@ -0,0 +1,27 @@ +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[project] +name = "racer-vllm" +version = "0.1.0" +description = "vLLM model loader that reads weights from a Racer cache" +readme = "README.md" +license = { text = "Apache-2.0" } +requires-python = ">=3.10" +dependencies = ["racer-client"] + +[project.optional-dependencies] +test = ["pytest"] + +[project.entry-points."vllm.general_plugins"] +racer = "racer_vllm:register" + +[project.scripts] +racer-vllm-fetch = "racer_vllm.model:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/clients/python/racer-vllm/src/racer_vllm/__init__.py b/clients/python/racer-vllm/src/racer_vllm/__init__.py new file mode 100644 index 0000000000..f1801da289 --- /dev/null +++ b/clients/python/racer-vllm/src/racer_vllm/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""vLLM plugin that loads model weights from a Racer cache.""" + +from .loader import LOAD_FORMAT, register +from .model import ModelURI, fetch, parse_uri + +__all__ = ["LOAD_FORMAT", "ModelURI", "fetch", "parse_uri", "register"] diff --git a/clients/python/racer-vllm/src/racer_vllm/loader.py b/clients/python/racer-vllm/src/racer_vllm/loader.py new file mode 100644 index 0000000000..e578482be2 --- /dev/null +++ b/clients/python/racer-vllm/src/racer_vllm/loader.py @@ -0,0 +1,97 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""vLLM model loader for --load-format racer. + +Targets vLLM v0.25.x. The loader interface used is +vllm/model_executor/model_loader/base_loader.py (BaseModelLoader with +download_model(model_config) and load_weights(model, model_config)), and +registration uses register_model_loader from +vllm/model_executor/model_loader/__init__.py. Recent main adds optional hooks +(get_external_weight_memory, create_model) with defaults, so this loader +works there too. +""" + +from __future__ import annotations + +import logging + +from .model import client_for, resolve_uri, weight_files +from .weights import DEFAULT_DEPTH, DEFAULT_STAGING_BYTES, TorchBackend, iter_weights + +log = logging.getLogger(__name__) + +LOAD_FORMAT = "racer" +_registered = False + + +def _settings(extra: dict) -> dict: + return { + "staging_bytes": int(extra.get("staging_bytes", DEFAULT_STAGING_BYTES)), + "concurrency": int(extra.get("concurrency", DEFAULT_DEPTH)), + "clone": bool(extra.get("clone", True)), + "socket": extra.get("socket"), + "use_vmm": bool(extra.get("use_vmm", True)), + } + + +def make_loader_class(base): + class RacerModelLoader(base): + """Loads safetensors weights from a Racer cache. + + model_loader_extra_config keys: + uri racer:////; optional when the + model directory has racer-model.json (racer-vllm-fetch) + staging_bytes size of each GPU staging buffer (default 256 MiB) + concurrency number of staging buffers and reads in flight (default 4) + clone copy each tensor out of staging (default true) + socket client socket path (else RACER_SOCKET, else the cache) + use_vmm allow exportable cuMemCreate staging (default true) + """ + + def __init__(self, load_config): + super().__init__(load_config) + self.extra = dict(getattr(load_config, "model_loader_extra_config", None) or {}) + self.settings = _settings(self.extra) + + def _client(self, model_config): + uri = resolve_uri(str(model_config.model), self.extra) + client = client_for(uri, self.settings["socket"], staging_bytes=self.settings["staging_bytes"]) + return uri, client + + def download_model(self, model_config) -> None: + # Nothing to download; check the weights are reachable. + uri, client = self._client(model_config) + with client: + weight_files(client, uri) + + def load_weights(self, model, model_config) -> None: + uri, client = self._client(model_config) + s = self.settings + with client: + backend = TorchBackend(clone=s["clone"], use_vmm=s["use_vmm"]) + weights = iter_weights(client, uri, backend, s["staging_bytes"], s["concurrency"]) + try: + loaded = model.load_weights(weights) + finally: + # Run the generator's cleanup (executor shutdown and GPU + # fences) before the client closes, even if load_weights + # raised or returned without exhausting the iterator. + weights.close() + if loaded is not None and getattr(model_config, "quantization", None) is None: + missing = {n for n, _ in model.named_parameters()} - set(loaded) + if missing: + raise ValueError(f"racer loader: weights not loaded: {sorted(missing)[:10]}") + + return RacerModelLoader + + +def register() -> None: + """vllm.general_plugins entry point. Safe to call more than once.""" + global _registered + if _registered: + return + from vllm.model_executor.model_loader import register_model_loader + from vllm.model_executor.model_loader.base_loader import BaseModelLoader + + register_model_loader(LOAD_FORMAT)(make_loader_class(BaseModelLoader)) + _registered = True diff --git a/clients/python/racer-vllm/src/racer_vllm/model.py b/clients/python/racer-vllm/src/racer_vllm/model.py new file mode 100644 index 0000000000..8eafc2dfba --- /dev/null +++ b/clients/python/racer-vllm/src/racer_vllm/model.py @@ -0,0 +1,168 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""racer:// model locations and small-file fetching.""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass + +import racer + +SCHEME = "racer://" +MARKER = "racer-model.json" +INDEX = "model.safetensors.index.json" +SINGLE = "model.safetensors" + +# Small files vLLM and transformers read to build the model and tokenizer. +CONFIG_FILES = ( + "config.json", + "generation_config.json", + "tokenizer.json", + "tokenizer_config.json", + "special_tokens_map.json", + "tokenizer.model", + "vocab.json", + "vocab.txt", + "merges.txt", + "added_tokens.json", + "chat_template.jinja", + "chat_template.json", + "preprocessor_config.json", + "processor_config.json", + "params.json", + "tekken.json", + INDEX, +) +REQUIRED_FILES = ("config.json",) +_SMALL_SUFFIXES = (".json", ".txt", ".model", ".jinja", ".py", ".tiktoken") + + +@dataclass(frozen=True) +class ModelURI: + cache: str + bucket: str + prefix: str # no leading or trailing slash; may be empty + + def key(self, name: str) -> str: + return f"{self.prefix}/{name}" if self.prefix else name + + def request(self, name: str) -> racer.Request: + return racer.object_request(self.bucket, self.key(name)) + + def __str__(self) -> str: + return f"{SCHEME}{self.cache}/{self.bucket}/{self.prefix}".rstrip("/") + + +def parse_uri(uri: str) -> ModelURI: + """Parse racer:////.""" + if not uri.startswith(SCHEME): + raise ValueError(f"not a racer:// URI: {uri!r}") + parts = uri[len(SCHEME) :].split("/", 2) + if len(parts) < 2 or not parts[0] or not parts[1]: + raise ValueError(f"racer URI needs a cache and a bucket: {uri!r}") + prefix = parts[2].strip("/") if len(parts) == 3 else "" + return ModelURI(parts[0], parts[1], prefix) + + +def client_for(uri: ModelURI, socket_path: str | None = None, **kw) -> racer.Client: + """Client for the URI's cache. RACER_SOCKET or socket_path override the path.""" + return racer.Client(cache=uri.cache, socket_path=socket_path, **kw) + + +def read_small(client: racer.Client, uri: ModelURI, name: str) -> bytes | None: + """Read a whole small object, or None when it does not exist.""" + try: + return client.read(uri.request(name)) + except racer.NotFound: + return None + + +def index_extra_files(index: dict) -> list: + """Small file names listed as string values in the index metadata.""" + out = [] + for v in (index.get("metadata") or {}).values(): + if isinstance(v, str) and v.endswith(_SMALL_SUFFIXES) and not v.startswith("/") and ".." not in v: + out.append(v) + return out + + +def weight_files(client: racer.Client, uri: ModelURI) -> list: + """List safetensors files from the index, else the single model.safetensors.""" + raw = read_small(client, uri, INDEX) + if raw is None: + client.stat(uri.request(SINGLE)) # raises NotFound with a clear key + return [SINGLE] + index = json.loads(raw) + files = sorted(set((index.get("weight_map") or {}).values())) + if not files: + raise ValueError(f"{INDEX} has an empty weight_map") + for f in files: + if f.startswith("/") or ".." in f.split("/"): + raise ValueError(f"unsafe file name in {INDEX}: {f!r}") + return files + + +def fetch(uri: str | ModelURI, dest: str, client: racer.Client | None = None, extra=()) -> list: + """Copy the small model files to dest and write the racer-model.json marker. + + Point vLLM at dest (--model dest, and --tokenizer dest if needed) with + --load-format racer. The loader reads the marker to find the weights. + """ + u = parse_uri(uri) if isinstance(uri, str) else uri + client = client or client_for(u) + os.makedirs(dest, exist_ok=True) + names = list(CONFIG_FILES) + list(extra) + fetched = [] + seen = set() + while names: + name = names.pop(0) + if name in seen: + continue + seen.add(name) + data = read_small(client, u, name) + if data is None: + if name in REQUIRED_FILES: + raise FileNotFoundError(f"{u.key(name)} not found in {u}") + continue + path = os.path.join(dest, name) + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = path + ".tmp" + with open(tmp, "wb") as f: + f.write(data) + os.replace(tmp, path) + fetched.append(name) + if name == INDEX: + names.extend(index_extra_files(json.loads(data))) + with open(os.path.join(dest, MARKER), "w") as f: + json.dump({"uri": str(u)}, f) + return fetched + + +def resolve_uri(model_path: str, extra: dict) -> ModelURI: + """Find the racer:// URI from extra config, the model name, or the marker.""" + if extra.get("uri"): + return parse_uri(extra["uri"]) + if model_path.startswith(SCHEME): + return parse_uri(model_path) + marker = os.path.join(model_path, MARKER) + if os.path.exists(marker): + with open(marker) as f: + return parse_uri(json.load(f)["uri"]) + raise ValueError( + "racer loader: no model URI; set model_loader_extra_config.uri or run racer-vllm-fetch into the model directory" + ) + + +def main(argv=None) -> int: + import argparse + + p = argparse.ArgumentParser(prog="racer-vllm-fetch", description="Fetch model config files from Racer.") + p.add_argument("uri", help="racer:////") + p.add_argument("dest", help="local directory") + p.add_argument("--file", action="append", default=[], help="extra small file to fetch") + a = p.parse_args(argv) + for name in fetch(a.uri, a.dest, extra=a.file): + print(name) + return 0 diff --git a/clients/python/racer-vllm/src/racer_vllm/safetensors.py b/clients/python/racer-vllm/src/racer_vllm/safetensors.py new file mode 100644 index 0000000000..2574e35a53 --- /dev/null +++ b/clients/python/racer-vllm/src/racer_vllm/safetensors.py @@ -0,0 +1,205 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Torch-free safetensors header parsing and read planning. + +Format: an 8-byte little-endian header length N, N bytes of JSON, then the +tensor data. Each JSON entry has dtype, shape and data_offsets [begin, end) +relative to the end of the header. "__metadata__" holds string metadata. +""" + +from __future__ import annotations + +import json +import math +import struct +from dataclasses import dataclass, field + +MAX_HEADER_BYTES = 100 * 1024 * 1024 + +# Bytes per element for each safetensors dtype. +DTYPE_SIZES = { + "BOOL": 1, + "U8": 1, + "I8": 1, + "F8_E4M3": 1, + "F8_E5M2": 1, + "F8_E8M0": 1, + "I16": 2, + "U16": 2, + "F16": 2, + "BF16": 2, + "I32": 4, + "U32": 4, + "F32": 4, + "I64": 8, + "U64": 8, + "F64": 8, +} + + +class SafetensorsError(ValueError): + pass + + +@dataclass(frozen=True) +class TensorInfo: + name: str + dtype: str + shape: tuple + start: int # absolute file offset + end: int + + @property + def nbytes(self) -> int: + return self.end - self.start + + @property + def itemsize(self) -> int: + return DTYPE_SIZES[self.dtype] + + +@dataclass +class Header: + data_start: int + tensors: list + metadata: dict = field(default_factory=dict) + + +def header_length(prefix: bytes) -> int: + if len(prefix) != 8: + raise SafetensorsError("short safetensors prefix") + (n,) = struct.unpack(" MAX_HEADER_BYTES: + raise SafetensorsError(f"bad safetensors header length {n}") + return n + + +def _is_uint_list(v) -> bool: + # JSON integers only: reject floats, numeric strings, and bools (bool is an int subclass). + return isinstance(v, list) and all(type(x) is int and x >= 0 for x in v) + + +def parse_header(raw: bytes, file_size: int | None = None) -> Header: + """Parse header JSON (without the 8-byte prefix) into sorted TensorInfo.""" + try: + doc = json.loads(raw.decode("utf-8").rstrip(" ")) + except (UnicodeDecodeError, ValueError) as exc: + raise SafetensorsError("bad safetensors header JSON") from exc + if not isinstance(doc, dict): + raise SafetensorsError("safetensors header is not an object") + data_start = 8 + len(raw) + metadata = doc.pop("__metadata__", None) or {} + tensors = [] + for name, spec in doc.items(): + try: + dtype = spec["dtype"] + raw_shape = spec["shape"] + raw_offsets = spec["data_offsets"] + except (KeyError, TypeError) as exc: + raise SafetensorsError(f"bad entry for {name!r}") from exc + if not _is_uint_list(raw_shape) or not _is_uint_list(raw_offsets) or len(raw_offsets) != 2: + raise SafetensorsError(f"bad shape or offsets for {name!r}") + shape = tuple(raw_shape) + begin, end = raw_offsets + if dtype not in DTYPE_SIZES: + raise SafetensorsError(f"unsupported dtype {dtype} for {name!r}") + if any(d < 0 for d in shape) or begin < 0 or end < begin: + raise SafetensorsError(f"bad shape or offsets for {name!r}") + if end - begin != math.prod(shape) * DTYPE_SIZES[dtype]: + raise SafetensorsError(f"size mismatch for {name!r}") + tensors.append(TensorInfo(name, dtype, shape, data_start + begin, data_start + end)) + tensors.sort(key=lambda t: (t.start, t.end)) + prev = data_start + for t in tensors: + if t.start < prev: + raise SafetensorsError(f"overlapping tensor {t.name!r}") + prev = t.end + if file_size is not None and prev > file_size: + raise SafetensorsError("tensor data past end of file") + return Header(data_start, tensors, metadata) + + +@dataclass(frozen=True) +class Part: + """Bytes [offset, offset + length) of a tensor that spans several chunks.""" + + tensor: TensorInfo + offset: int + length: int + + @property + def last(self) -> bool: + return self.offset + self.length == self.tensor.nbytes + + +@dataclass +class Chunk: + """A contiguous file range read with one request.""" + + start: int + end: int + tensors: list # tensors that lie wholly inside the chunk + dedicated: bool = False # larger than the staging buffer + parts: list = field(default_factory=list) # Parts of tensors split across chunks + + @property + def length(self) -> int: + return self.end - self.start + + +def plan_chunks(tensors: list, staging_bytes: int, page_bytes: int = 0) -> list: + """Group sorted tensors into ranges of at most staging_bytes. + + A tensor bigger than staging_bytes gets its own dedicated chunk. Zero-size + tensors join the current chunk and need no bytes. + + With page_bytes no larger than staging_bytes, a chunk that would end in + the middle of a page ends on the page boundary instead, and the tensor that + crosses the boundary is split into Parts read by consecutive chunks. Without + this, both chunks read the boundary page. + """ + paged = 0 < page_bytes <= staging_bytes + chunks: list = [] + cur: Chunk | None = None + limit = 0 + + def open_chunk(start: int) -> Chunk: + nonlocal limit + limit = start + staging_bytes + if paged: + limit -= limit % page_bytes + return Chunk(start, start, []) + + for t in tensors: + if t.nbytes > staging_bytes: + if cur is not None: + chunks.append(cur) + cur = None + chunks.append(Chunk(t.start, t.end, [t], dedicated=True)) + continue + if cur is not None and t.end > limit and (t.start >= limit or not paged): + chunks.append(cur) + cur = None + if cur is None: + cur = open_chunk(t.start) + if t.end <= limit: + cur.tensors.append(t) + cur.end = max(cur.end, t.end) + continue + pos = t.start + while t.end > limit: + cur.parts.append(Part(t, pos - t.start, limit - pos)) + cur.end = pos = limit + chunks.append(cur) + cur = open_chunk(pos) + cur.parts.append(Part(t, pos - t.start, t.end - pos)) + cur.end = t.end + if cur is not None: + chunks.append(cur) + return chunks + + +def tensor_bytes(buf, chunk: Chunk, t: TensorInfo) -> memoryview: + """Return the bytes of t inside a buffer holding chunk (torch-free).""" + rel = t.start - chunk.start + return memoryview(buf)[rel : rel + t.nbytes] diff --git a/clients/python/racer-vllm/src/racer_vllm/weights.py b/clients/python/racer-vllm/src/racer_vllm/weights.py new file mode 100644 index 0000000000..5637aeec18 --- /dev/null +++ b/clients/python/racer-vllm/src/racer_vllm/weights.py @@ -0,0 +1,293 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Stream safetensors weights from Racer into GPU staging buffers. + +The pipeline is torch-free; a Backend supplies memory and tensor views. +""" + +from __future__ import annotations + +import collections +import logging +from concurrent.futures import ThreadPoolExecutor +from typing import Iterator + +import racer + +from .model import ModelURI, weight_files +from .safetensors import Chunk, TensorInfo, header_length, parse_header, plan_chunks + +log = logging.getLogger(__name__) + +DEFAULT_STAGING_BYTES = 256 * 1024 * 1024 +DEFAULT_DEPTH = 4 + + +class Backend: + """Memory and tensor views for the pipeline.""" + + def new_staging(self, size: int): + raise NotImplementedError + + def new_dedicated(self, size: int): + raise NotImplementedError + + def tensor(self, target, rel: int, info: TensorInfo, staged: bool): + raise NotImplementedError + + def new_joined(self, info: TensorInfo): + """Memory for a tensor that is split across chunks.""" + raise NotImplementedError + + def copy_part(self, joined, offset: int, target, rel: int, length: int) -> None: + """Copy length bytes at rel in target to offset in joined.""" + raise NotImplementedError + + def joined_tensor(self, joined, info: TensorInfo): + """Return the tensor once every part has been copied into joined.""" + raise NotImplementedError + + def done(self, target) -> None: + """The consumer is finished with every tensor from target.""" + + def wait(self, target) -> None: + """Block until work queued by the consumer on target has finished.""" + + +def _file_chunks(client: racer.Client, uri: ModelURI, staging_bytes: int, page_bytes: int): + for name in weight_files(client, uri): + req = uri.request(name) + meta = client.stat(req) + prefix = client.read(req, 0, 8, if_match=meta.etag) + n = header_length(prefix) + header = parse_header(client.read(req, 8, n, if_match=meta.etag), meta.size) + log.debug("racer: %s has %d tensors", name, len(header.tensors)) + for chunk in plan_chunks(header.tensors, staging_bytes, page_bytes): + yield req, meta.etag, chunk + + +def iter_weights( + client: racer.Client, + uri: ModelURI, + backend: Backend, + staging_bytes: int = DEFAULT_STAGING_BYTES, + depth: int = DEFAULT_DEPTH, + page_bytes: int = racer.PAGE_SIZE, +) -> Iterator[tuple]: + """Yield (name, tensor) for every tensor in the model. + + Up to ``depth`` chunks are read ahead, each into its own staging buffer, + so reads overlap with the consumer. A staging buffer is reused only after + the consumer has moved past all of its tensors and backend.wait() says + queued GPU work that reads it has finished. Every staging and dedicated + target is fenced with done() and wait() before the pipeline drops its + reference, including on early close or error, so external allocations + are never released while queued GPU work may still read them. Dedicated + targets are fenced and dropped as soon as the consumer resumes past + them, so oversized tensors do not accumulate in GPU memory. + + Chunks end on Racer page boundaries (page_bytes) where they can, so + neighboring chunks do not both read the boundary page. A tensor that + crosses a boundary is copied out of each chunk into memory from + backend.new_joined() and yielded with the chunk that holds its last byte. + """ + if depth < 1: + raise ValueError("depth must be at least 1") + free = [backend.new_staging(staging_bytes) for _ in range(depth)] + chunks = _file_chunks(client, uri, staging_bytes, page_bytes) + joined: dict = {} + inflight: collections.deque = collections.deque() + pool = ThreadPoolExecutor(max_workers=depth, thread_name_prefix="racer-read") + + def read(req, etag, chunk: Chunk, target): + backend.wait(target) + if chunk.length: + client.read_into_gpu(req, target, 0, offset=chunk.start, length=chunk.length, if_match=etag) + return target + + def fill() -> bool: + while len(inflight) < depth: + nxt = next(chunks, None) + if nxt is None: + return bool(inflight) + req, etag, chunk = nxt + if chunk.dedicated: + target = backend.new_dedicated(chunk.length) + else: + target = free.pop() + inflight.append((pool.submit(read, req, etag, chunk, target), chunk)) + return True + + active = None + try: + while fill(): + fut, chunk = inflight.popleft() + target = fut.result() + active = target + for t in chunk.tensors: + yield t.name, backend.tensor(target, t.start - chunk.start, t, not chunk.dedicated) + for part in chunk.parts: + t = part.tensor + if part.offset == 0: + joined[t.name] = backend.new_joined(t) + rel = t.start + part.offset - chunk.start + backend.copy_part(joined[t.name], part.offset, target, rel, part.length) + if part.last: + yield t.name, backend.joined_tensor(joined.pop(t.name), t) + active = None + backend.done(target) + if chunk.dedicated: + # Fence and drop now so oversized targets do not accumulate; + # tensors the consumer kept hold their own storage reference. + backend.wait(target) + del target + else: + free.append(target) + finally: + for fut, _ in inflight: + fut.cancel() + pool.shutdown(wait=True) + if active is not None: + backend.done(active) + backend.wait(active) + # Reads have stopped; fence consumer GPU work before release. Targets + # still in flight were fenced by read() and the consumer never saw them. + for target in free: + backend.wait(target) + + +class MemoryBackend(Backend): + """Host memory backend: plain buffers, bytes views. Used for tests and CPU.""" + + class Target: + def __init__(self, size: int) -> None: + self.size = size + self.buf = bytearray(size) + + def copy_from_host(self, offset: int, data) -> None: + mv = memoryview(data).cast("B") + self.buf[offset : offset + len(mv)] = mv + + def new_staging(self, size: int): + return self.Target(size) + + def new_dedicated(self, size: int): + return self.Target(size) + + def tensor(self, target, rel, info, staged): + return bytes(target.buf[rel : rel + info.nbytes]) + + def new_joined(self, info): + return bytearray(info.nbytes) + + def copy_part(self, joined, offset, target, rel, length): + joined[offset : offset + length] = target.buf[rel : rel + length] + + def joined_tensor(self, joined, info): + return bytes(joined) + + +TORCH_DTYPES = { + "BOOL": "bool", + "U8": "uint8", + "I8": "int8", + "F8_E4M3": "float8_e4m3fn", + "F8_E5M2": "float8_e5m2", + "F8_E8M0": "float8_e8m0fnu", + "I16": "int16", + "U16": "uint16", + "F16": "float16", + "BF16": "bfloat16", + "I32": "int32", + "U32": "uint32", + "F32": "float32", + "I64": "int64", + "U64": "uint64", + "F64": "float64", +} + + +class TorchBackend(Backend): + """CUDA backend. + + Staging buffers are exportable cuMemCreate allocations when the CUDA + driver allows it (so gpu-read works), else plain torch tensors (host or + stream mode plus a copy). Tensors from staging buffers are cloned unless + clone=False; then they are views that are valid only until the consumer + asks for the next chunk's tensors. + """ + + def __init__(self, device: int | None = None, clone: bool = True, use_vmm: bool = True) -> None: + import torch + + self.torch = torch + self.device = torch.cuda.current_device() if device is None else device + self.clone = clone + self.use_vmm = use_vmm + self._flat: dict = {} + self._events: dict = {} + + def _new(self, size: int): + import racer.cuda as cuda + import racer.tensors as tensors + + if self.use_vmm: + try: + alloc = cuda.GpuAllocation(size, self.device) + self._flat[id(alloc)] = tensors.as_tensor(alloc) + return alloc + except racer.RacerError as exc: + log.info("racer: exportable GPU memory unavailable (%s); using torch memory", exc) + self.use_vmm = False + t = self.torch.empty(size, dtype=self.torch.uint8, device=f"cuda:{self.device}") + target = tensors.TensorTarget(t) + self._flat[id(target)] = t + return target + + def new_staging(self, size): + return self._new(size) + + def new_dedicated(self, size): + return self._new(size) + + def tensor(self, target, rel, info, staged): + torch = self.torch + dtype = getattr(torch, TORCH_DTYPES[info.dtype]) + if info.nbytes == 0: + return torch.empty(info.shape, dtype=dtype, device=f"cuda:{self.device}") + flat = self._flat[id(target)] + if not staged: + # Dedicated memory: the tensor keeps it alive; drop our reference. + self._flat.pop(id(target), None) + raw = flat[rel : rel + info.nbytes] + if rel % info.itemsize: + raw = raw.clone() # realign; the clone owns its memory + elif staged and self.clone: + raw = raw.clone() + return raw.view(dtype).view(info.shape) + + def new_joined(self, info): + return self.torch.empty(info.nbytes, dtype=self.torch.uint8, device=f"cuda:{self.device}") + + def copy_part(self, joined, offset, target, rel, length): + # Queued before done(target), so the staging buffer is not reused + # until the copy has finished. + joined[offset : offset + length].copy_(self._flat[id(target)][rel : rel + length]) + + def joined_tensor(self, joined, info): + dtype = getattr(self.torch, TORCH_DTYPES[info.dtype]) + return joined.view(dtype).view(info.shape) + + def done(self, target): + # Record on the target device's stream; the calling thread's current + # device may differ, and its stream would not cover our copies. + cuda = self.torch.cuda + with cuda.device(self.device): + ev = cuda.Event() + ev.record(cuda.current_stream(self.device)) + self._events[id(target)] = ev + + def wait(self, target): + ev = self._events.pop(id(target), None) + if ev is not None: + ev.synchronize() diff --git a/clients/python/racer-vllm/tests/conftest.py b/clients/python/racer-vllm/tests/conftest.py new file mode 100644 index 0000000000..ea8395a24c --- /dev/null +++ b/clients/python/racer-vllm/tests/conftest.py @@ -0,0 +1,33 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +import json +import os +import struct +import sys + +HERE = os.path.dirname(__file__) +sys.path.insert(0, os.path.join(HERE, "..", "..", "racer", "tests")) + +import importlib.util # noqa: E402 + +# Reuse the racer client's fake server fixtures without a module name clash. +_spec = importlib.util.spec_from_file_location( + "racer_test_fixtures", os.path.join(HERE, "..", "..", "racer", "tests", "conftest.py") +) +_fixtures = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_fixtures) +server = _fixtures.server +small_pages = _fixtures.small_pages + + +def make_safetensors(tensors, metadata=None, pad=0): + """tensors: list of (name, dtype, shape, bytes). Returns file bytes.""" + header = {} + data = b"" + for name, dtype, shape, raw in tensors: + header[name] = {"dtype": dtype, "shape": list(shape), "data_offsets": [len(data), len(data) + len(raw)]} + data += raw + if metadata: + header["__metadata__"] = metadata + h = json.dumps(header).encode() + b" " * pad + return struct.pack("= 2 + + +class _Param: + pass +class _Param: + pass + + +def install_fake_vllm(monkeypatch): + registry = {} + vllm = types.ModuleType("vllm") + me = types.ModuleType("vllm.model_executor") + ml = types.ModuleType("vllm.model_executor.model_loader") + bl = types.ModuleType("vllm.model_executor.model_loader.base_loader") + + class BaseModelLoader: + def __init__(self, load_config): + self.load_config = load_config + + def register_model_loader(name): + def wrap(cls): + assert issubclass(cls, BaseModelLoader) + registry[name] = cls + return cls + + return wrap + + ml.register_model_loader = register_model_loader + bl.BaseModelLoader = BaseModelLoader + for name, mod in { + "vllm": vllm, + "vllm.model_executor": me, + "vllm.model_executor.model_loader": ml, + "vllm.model_executor.model_loader.base_loader": bl, + }.items(): + monkeypatch.setitem(sys.modules, name, mod) + monkeypatch.setattr(loader, "_registered", False) + return registry + + +def test_register_is_idempotent(monkeypatch): + install_fake_vllm(monkeypatch) + ml = sys.modules["vllm.model_executor.model_loader"] + inner = ml.register_model_loader + calls = [] + + def counting(name): + calls.append(name) + return inner(name) + + ml.register_model_loader = counting + loader.register() + loader.register() + assert calls == ["racer"] + assert loader._registered is True + + +def test_register_and_load(server, uri, monkeypatch, tmp_path): + registry = install_fake_vllm(monkeypatch) + loader.register() + loader.register() + cls = registry["racer"] + setup_model(server, uri) + model.fetch(uri, str(tmp_path / "m"), client=racer.Client(socket_path=server.path)) + monkeypatch.setenv("RACER_SOCKET", server.path) + cfg = types.SimpleNamespace(model_loader_extra_config={"staging_bytes": 400, "concurrency": 2}) + inst = cls(cfg) + mc = types.SimpleNamespace(model=str(tmp_path / "m"), quantization=None) + inst.download_model(mc) + + # Swap the torch backend for the memory one, as there is no GPU here. + monkeypatch.setattr(loader, "TorchBackend", lambda **kw: MemoryBackend()) + + class Model: + def __init__(self): + self.got = {} + + def load_weights(self, weights): + for name, t in weights: + self.got[name] = t + return set(self.got) + + def named_parameters(self): + return [(n, _Param()) for n in ("w1", "w2", "w3", "w4")] + + m = Model() + inst.load_weights(m, mc) + assert m.got["w3"] == b"\3" * 900 + + class Missing(Model): + def named_parameters(self): + return super().named_parameters() + [("w5", _Param())] + + with pytest.raises(ValueError, match="w5"): + inst.load_weights(Missing(), mc) + + +def test_entry_point_declared(): + here = os.path.dirname(__file__) + with open(os.path.join(here, "..", "pyproject.toml"), encoding="utf-8") as f: + text = f.read() + assert '[project.entry-points."vllm.general_plugins"]' in text + assert 'racer = "racer_vllm:register"' in text + + +def test_torch_backend_records_event_on_target_device_stream(monkeypatch): + calls = [] + + class Event: + def record(self, stream=None): + calls.append(("record", stream)) + + def synchronize(self): + calls.append(("sync",)) + + class DeviceCtx: + def __init__(self, dev): + self.dev = dev + + def __enter__(self): + calls.append(("enter", self.dev)) + + def __exit__(self, *a): + calls.append(("exit", self.dev)) + + cuda = types.SimpleNamespace( + current_device=lambda: 0, + device=DeviceCtx, + Event=lambda: (calls.append(("event",)), Event())[1], + current_stream=lambda dev=None: ("stream", dev), + ) + monkeypatch.setitem(sys.modules, "torch", types.SimpleNamespace(cuda=cuda)) + from racer_vllm.weights import TorchBackend + + b = TorchBackend(device=3) + target = object() + b.done(target) + b.wait(target) + assert calls == [("enter", 3), ("event",), ("record", ("stream", 3)), ("exit", 3), ("sync",)] + + +class _FencingBackend(MemoryBackend): + """Records done/wait so tests can check every target is fenced.""" + + def __init__(self): + self.targets = [] + self.done_ids = set() + self.waited = set() + + def _track(self, t): + self.targets.append(t) + return t + + def new_staging(self, size): + return self._track(self.Target(size)) + + def new_dedicated(self, size): + return self._track(self.Target(size)) + + def done(self, target): + self.done_ids.add(id(target)) + self.waited.discard(id(target)) + + def wait(self, target): + if id(target) in self.done_ids: + self.waited.add(id(target)) + + def used(self): + return [t for t in self.targets if id(t) in self.done_ids] + + def assert_fenced(self): + assert self.used() + for t in self.used(): + assert id(t) in self.waited + + +@pytest.mark.parametrize("depth", [1, 2]) +def test_iter_weights_fences_on_completion(server, uri, depth): + setup_model(server, uri) + c = racer.Client(socket_path=server.path, staging_bytes=4096) + b = _FencingBackend() + names = [n for n, _ in iter_weights(c, uri, b, staging_bytes=400, depth=depth)] + assert "w3" in names + b.assert_fenced() + # The dedicated target (w3) was fenced too. + assert any(t.size == 900 for t in b.used()) + + +@pytest.mark.parametrize("stop", ["w1", "w3"]) +def test_iter_weights_fences_on_early_close(server, uri, stop): + setup_model(server, uri) + c = racer.Client(socket_path=server.path, staging_bytes=4096) + b = _FencingBackend() + gen = iter_weights(c, uri, b, staging_bytes=400, depth=2) + for name, _ in gen: + if name == stop: + break + gen.close() + b.assert_fenced() + + +def test_iter_weights_fences_on_exception(server, uri): + setup_model(server, uri) + c = racer.Client(socket_path=server.path, staging_bytes=4096) + b = _FencingBackend() + gen = iter_weights(c, uri, b, staging_bytes=400, depth=2) + next(gen) + with pytest.raises(RuntimeError): + gen.throw(RuntimeError("consumer failed")) + b.assert_fenced() + + +@pytest.mark.parametrize("mode", ["raise", "early"]) +def test_load_weights_closes_iterator_before_client(server, uri, monkeypatch, tmp_path, mode): + registry = install_fake_vllm(monkeypatch) + loader.register() + setup_model(server, uri) + model.fetch(uri, str(tmp_path / "m"), client=racer.Client(socket_path=server.path)) + monkeypatch.setenv("RACER_SOCKET", server.path) + monkeypatch.setattr(loader, "TorchBackend", lambda **kw: MemoryBackend()) + events = [] + real_iter = loader.iter_weights + + def tracking_iter(*a, **kw): + try: + yield from real_iter(*a, **kw) + finally: + events.append("weights closed") + + monkeypatch.setattr(loader, "iter_weights", tracking_iter) + real_client_for = loader.client_for + + def tracking_client_for(*a, **kw): + c = real_client_for(*a, **kw) + inner_exit = c.__exit__ + + class Wrap: + def __getattr__(self, n): + return getattr(c, n) + + def __enter__(self): + c.__enter__() + return self + + def __exit__(self, *exc): + events.append("client closed") + return inner_exit(*exc) + + return Wrap() + + monkeypatch.setattr(loader, "client_for", tracking_client_for) + inst = registry["racer"](types.SimpleNamespace(model_loader_extra_config={"staging_bytes": 400})) + mc = types.SimpleNamespace(model=str(tmp_path / "m"), quantization=None) + + class Model: + def load_weights(self, weights): + next(weights) + if mode == "raise": + raise RuntimeError("boom") + return None + + if mode == "raise": + with pytest.raises(RuntimeError, match="boom"): + inst.load_weights(Model(), mc) + else: + inst.load_weights(Model(), mc) + assert events == ["weights closed", "client closed"] + + +def test_iter_weights_releases_dedicated_when_consumed(server, uri): + import gc + import weakref + + setup_model(server, uri) + c = racer.Client(socket_path=server.path, staging_bytes=4096) + refs = [] + + class _WeakBackend(_FencingBackend): + def new_dedicated(self, size): + t = self.Target(size) + refs.append(weakref.ref(t)) + return t + + b = _WeakBackend() + gen = iter_weights(c, uri, b, staging_bytes=400, depth=2) + for name, _ in gen: + if name == "w3": + break + assert len(refs) == 1 and refs[0]() is not None + target_id = id(refs[0]()) + # Resuming past w3 fences the dedicated target and drops it. + next(gen, None) + assert target_id in b.waited + gc.collect() + assert refs[0]() is None + gen.close() diff --git a/clients/python/racer-vllm/tests/test_safetensors.py b/clients/python/racer-vllm/tests/test_safetensors.py new file mode 100644 index 0000000000..e57988740b --- /dev/null +++ b/clients/python/racer-vllm/tests/test_safetensors.py @@ -0,0 +1,158 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +import struct + +import pytest +from conftest import make_safetensors + +from racer_vllm.safetensors import ( + SafetensorsError, + header_length, + parse_header, + plan_chunks, + tensor_bytes, +) + + +def sample(): + return make_safetensors( + [ + ("a", "F32", (2, 3), struct.pack("<6f", *range(6))), + ("b", "BF16", (4,), b"\1\2" * 4), + ("c", "U8", (0,), b""), + ("d", "I64", (), struct.pack(" start else set() + + +def test_plan_chunks_on_page_boundaries(): + f = make_safetensors([(f"t{i}", "U8", (90,), bytes([i]) * 90) for i in range(8)] + [("z", "U8", (0,), b"")]) + n = header_length(f[:8]) + h = parse_header(f[8 : 8 + n]) + chunks = plan_chunks(h.tensors, 250, 64) + # Every chunk but the last ends on a page boundary, and no two chunks share a page. + assert all(c.end % 64 == 0 for c in chunks[:-1]) + assert all(c.length <= 250 for c in chunks) + seen = set() + for c in chunks: + pages = _pages(c.start, c.end, 64) + assert not pages & seen + seen |= pages + # Each tensor is either whole in one chunk or covered exactly once by its parts. + covered = {t.name: [] for t in h.tensors} + for c in chunks: + for t in c.tensors: + assert c.start <= t.start and t.end <= c.end + covered[t.name].append((0, t.nbytes)) + for p in c.parts: + assert c.start <= p.tensor.start + p.offset and p.tensor.start + p.offset + p.length <= c.end + covered[p.tensor.name].append((p.offset, p.offset + p.length)) + for t in h.tensors: + spans = sorted(covered[t.name]) + assert spans[0][0] == 0 and spans[-1][1] == t.nbytes, t.name + assert all(a[1] == b[0] for a, b in zip(spans, spans[1:])), t.name + assert any(c.parts for c in chunks) + assert [p.last for c in chunks for p in c.parts].count(True) == len({p.tensor.name for c in chunks for p in c.parts}) + + +def test_plan_chunks_splits_across_several_chunks(): + # With staging of just under two pages, each chunk after the first holds + # one page, so a tensor that starts mid-chunk spans three chunks. + f = make_safetensors([("a", "U8", (100,), b"a" * 100), ("b", "U8", (127,), b"b" * 127)]) + n = header_length(f[:8]) + h = parse_header(f[8 : 8 + n]) + chunks = plan_chunks(h.tensors, 127, 64) + parts = [p for c in chunks for p in c.parts if p.tensor.name == "b"] + assert len(parts) >= 3 + assert [(p.offset, p.length) for p in parts][0][0] == 0 + assert all(a.offset + a.length == b.offset for a, b in zip(parts, parts[1:])) + assert sum(p.length for p in parts) == 127 + assert [p.last for p in parts] == [False] * (len(parts) - 1) + [True] + assert all(c.length <= 127 for c in chunks) + + +def test_plan_chunks_ignores_pages_larger_than_staging(): + f = make_safetensors([(f"t{i}", "U8", (100,), bytes([i]) * 100) for i in range(5)]) + n = header_length(f[:8]) + h = parse_header(f[8 : 8 + n]) + assert [[t.name for t in c.tensors] for c in plan_chunks(h.tensors, 250, 1024)] == [ + [t.name for t in c.tensors] for c in plan_chunks(h.tensors, 250) + ] + assert not any(c.parts for c in plan_chunks(h.tensors, 250, 1024)) + + +@pytest.mark.parametrize( + "header", + [ + b"[]", + b"{not json", + b'{"a":{"dtype":"F4","shape":[1],"data_offsets":[0,1]}}', + b'{"a":{"dtype":"F32","shape":[2],"data_offsets":[0,4]}}', + b'{"a":{"dtype":"U8","shape":[2],"data_offsets":[0,2]},"b":{"dtype":"U8","shape":[2],"data_offsets":[1,3]}}', + b'{"a":{"dtype":"U8","shape":[-1],"data_offsets":[0,0]}}', + b'{"a":{"dtype":"U8"}}', + b'{"a":{"dtype":"U8","shape":[1.9],"data_offsets":[0.9,1.9]}}', + b'{"a":{"dtype":"U8","shape":[1],"data_offsets":[0.0,1.0]}}', + b'{"a":{"dtype":"U8","shape":[-0.5],"data_offsets":[0,0]}}', + b'{"a":{"dtype":"U8","shape":["1"],"data_offsets":[0,1]}}', + b'{"a":{"dtype":"U8","shape":[1],"data_offsets":["0","1"]}}', + b'{"a":{"dtype":"U8","shape":[true],"data_offsets":[0,1]}}', + b'{"a":{"dtype":"U8","shape":[1],"data_offsets":[false,true]}}', + b'{"a":{"dtype":"U8","shape":"1","data_offsets":[0,1]}}', + b'{"a":{"dtype":"U8","shape":[1],"data_offsets":[0,1,2]}}', + b'{"a":{"dtype":"U8","shape":[1],"data_offsets":{"0":0}}}', + b'{"a":[]}', + ], +) +def test_bad_headers(header): + with pytest.raises(SafetensorsError): + parse_header(header) + + +def test_bad_length(): + with pytest.raises(SafetensorsError): + header_length(struct.pack("/client/socket`. + +## Reading into host memory + +```python +with racer.HostBuffer(meta.size) as buf: + client.read_into_host(request, buf) +``` + +`HostBuffer` is a `memfd`. The client passes it to the dataplane with +`SCM_RIGHTS` (`host-read`), and the dataplane writes straight into it. Older +dataplanes do not know `host-read`; the client then uses the normal stream. + +## Reading into GPU memory + +```python +from racer import cuda, tensors + +alloc = cuda.GpuAllocation(meta.size, device=0) +client.read_into_gpu(request, alloc) +t = tensors.as_tensor(alloc) # uint8 CUDA tensor, no copy +``` + +Modes are tried in order: `gpu-read`, then `host-read` into a pinned staging +buffer plus a copy to the GPU, then the stream plus a copy. When a dataplane +says a mode is not supported, the client remembers that for the socket path +until the process exits. + +`GpuAllocation` uses the CUDA driver API (`libcuda.so.1`, through ctypes). +Set `RACER_LIBCUDA` to load a different library. torch is needed only for +`racer.tensors`. A tensor from `as_tensor` keeps the allocation alive. + +## Object keys + +`object_key(bucket, key, version_id)` gives the same 64-hex key as +racer-object (`internal/racerobject/racerobject.go`). + +## Tests + +``` +make racer-python-test +``` diff --git a/clients/python/racer/pyproject.toml b/clients/python/racer/pyproject.toml new file mode 100644 index 0000000000..6c0349f101 --- /dev/null +++ b/clients/python/racer/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[project] +name = "racer-client" +version = "0.1.0" +description = "Python client for the Racer node-local object cache" +readme = "README.md" +license = { text = "Apache-2.0" } +requires-python = ">=3.10" +dependencies = [] + +[project.optional-dependencies] +torch = ["torch"] +test = ["pytest"] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/clients/python/racer/src/racer/__init__.py b/clients/python/racer/src/racer/__init__.py new file mode 100644 index 0000000000..2a338da74e --- /dev/null +++ b/clients/python/racer/src/racer/__init__.py @@ -0,0 +1,50 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Python client for Racer, the Unbounded node-local object cache.""" + +from .client import ( + GPU, + HOST, + STREAM, + Client, + Metadata, + ReadResult, + forget_modes, + socket_path_for, +) +from .errors import ( + NotFound, + ProtocolError, + RacerError, + StatusError, + TruncatedError, + UnsatisfiableRange, + VersionUnavailable, +) +from ._wire import PAGE_SIZE +from .host import HostBuffer +from .keys import Request, canonical_json, object_key, object_request + +__all__ = [ + "GPU", + "HOST", + "PAGE_SIZE", + "STREAM", + "Client", + "HostBuffer", + "Metadata", + "NotFound", + "ProtocolError", + "RacerError", + "ReadResult", + "Request", + "StatusError", + "TruncatedError", + "UnsatisfiableRange", + "VersionUnavailable", + "canonical_json", + "forget_modes", + "object_key", + "object_request", + "socket_path_for", +] diff --git a/clients/python/racer/src/racer/_wire.py b/clients/python/racer/src/racer/_wire.py new file mode 100644 index 0000000000..5fa513beba --- /dev/null +++ b/clients/python/racer/src/racer/_wire.py @@ -0,0 +1,221 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Racer client socket wire format. + +This mirrors the Go SDK in pkg/racersdk/internal/wire/wire.go and the +dataplane in cmd/racer-dataplane/object-wire/src/client.rs. +""" + +from __future__ import annotations + +import struct +from dataclasses import dataclass + +from .errors import ProtocolError, RacerError, status_error + +MAX_HEAD_BYTES = 32 * 1024 +MAX_FIELD_BYTES = 8192 +OBJECT_PREFIX = "/v1/objects/" +PAGE_SIZE = 16 * 1024 * 1024 +FRAME_SIZE = 21 +CREDIT_SIZE = 12 +PAGE_FRAME = 1 +COMPLETE_FRAME = 2 +MAX_WIRE_INTEGER = (1 << 63) - 1 + +# Credits the Go SDK grants per Get (pkg/racersdk/client.go: pageCredits = 2). +PAGE_CREDITS = 2 +BYTE_CREDITS = PAGE_CREDITS * PAGE_SIZE + +_FRAME = struct.Struct(">BQQI") +_CREDIT = struct.Struct(">QI") + + +@dataclass(frozen=True) +class Frame: + kind: int + number: int + offset: int + length: int + + +def decode_frame(raw: bytes) -> Frame: + if len(raw) != FRAME_SIZE: + raise ProtocolError("short frame") + return Frame(*_FRAME.unpack(raw)) + + +def encode_frame(kind: int, number: int, offset: int, length: int) -> bytes: + return _FRAME.pack(kind, number, offset, length) + + +def encode_credit(number: int, length: int) -> bytes: + return _CREDIT.pack(number, length) + + +def decode_credit(raw: bytes) -> tuple[int, int]: + return _CREDIT.unpack(raw) + + +def page_count(first: int, end: int) -> int: + if end <= first: + return 0 + return (end - 1) // PAGE_SIZE - first // PAGE_SIZE + 1 + + +class Sequence: + """Validates page and completion frames like wire.Sequence.Accept. + + Pages must arrive strictly in page order, so tracking the count of + delivered pages is enough to reject duplicates. + """ + + def __init__(self, first: int, end: int) -> None: + self.first = first + self.end = end + self.pages = page_count(first, end) + self.delivered = 0 + self.bytes = 0 + self.complete = False + + def accept(self, f: Frame) -> None: + if self.complete: + raise ProtocolError("frame after completion") + if f.kind == COMPLETE_FRAME: + if ( + f.number != self.pages + or self.delivered != self.pages + or f.offset != self.end - self.first + or self.bytes != f.offset + or f.length != 0 + ): + raise ProtocolError("invalid completion frame") + self.complete = True + return + if ( + f.kind != PAGE_FRAME + or f.length == 0 + or f.offset < self.first + or f.offset >= self.end + or f.number != f.offset // PAGE_SIZE + ): + raise ProtocolError("invalid page frame") + start = max(self.first, f.number * PAGE_SIZE) + end = min(self.end, (f.number + 1) * PAGE_SIZE) + if ( + f.offset != start + or f.length != end - start + or f.number != self.first // PAGE_SIZE + self.delivered + ): + raise ProtocolError("page frame out of order") + self.delivered += 1 + self.bytes += f.length + + +def decimal(s: str | None) -> int: + """Parse a canonical non-negative int63 like wire.Decimal.""" + if s is None or s == "" or len(s) > 19 or (len(s) > 1 and s[0] == "0"): + raise ProtocolError("invalid decimal") + if not all("0" <= c <= "9" for c in s): + raise ProtocolError("invalid decimal") + n = int(s) + if n > MAX_WIRE_INTEGER: + raise ProtocolError("invalid decimal") + return n + + +def validate_etag(s: str) -> None: + if len(s) < 2 or len(s) > MAX_FIELD_BYTES or s[0] != '"' or s[-1] != '"': + raise RacerError("invalid etag") + for c in s[1:-1]: + o = ord(c) + if o != 0x21 and not 0x23 <= o <= 0x7E: + raise RacerError("invalid etag") + + +def validate_opaque(s: str) -> None: + raw = s.encode() + if len(raw) > MAX_FIELD_BYTES: + raise RacerError("header field too large") + if not raw or raw[0] == 0x20 or raw[-1] == 0x20 or any(b < 0x20 or b == 0x7F for b in raw): + raise RacerError("invalid header field") + + +def validate_content_type(s: str) -> bool: + return 0 < len(s) <= 256 and s.strip() == s and all(0x20 <= ord(c) <= 0x7E for c in s) + + +@dataclass +class ResponseHead: + status: int + headers: dict[str, str] # lower-case names + + def get(self, name: str) -> str | None: + return self.headers.get(name.lower()) + + +_SINGLETONS = { + "content-length", + "content-type", + "content-range", + "etag", + "racer-expires-at", + "racer-content-type", + "racer-object-length", + "racer-range-start", + "racer-range-end", + "racer-retain-until", + "racer-delivery", +} + + +def parse_response_head(raw: bytes) -> ResponseHead: + """Parse an HTTP/1.1 response head ending in CRLF CRLF.""" + try: + text = raw.decode("latin-1") + except Exception as exc: # pragma: no cover - latin-1 never fails + raise ProtocolError("bad head") from exc + lines = text.split("\r\n") + if len(lines) < 3 or lines[-1] != "" or lines[-2] != "": + raise ProtocolError("bad head") + line = lines[0] + if len(line) < 12 or not line.startswith("HTTP/1.1 ") or (len(line) > 12 and line[12] != " "): + raise ProtocolError("bad status line") + code = line[9:12] + if not code.isdigit(): + raise ProtocolError("bad status line") + headers: dict[str, str] = {} + for h in lines[1:-2]: + name, sep, value = h.partition(":") + if not sep or not name or name != name.strip(): + raise ProtocolError("bad header") + key = name.lower() + value = value.strip(" \t") + if key in headers: + if key in _SINGLETONS: + raise ProtocolError("duplicate header " + name) + headers[key] += ", " + value + else: + headers[key] = value + return ResponseHead(int(code), headers) + + +def connection_close(h: ResponseHead) -> bool: + v = h.get("connection") or "" + return any(t.strip().lower() == "close" for t in v.split(",")) + + +def check_error_status(h: ResponseHead) -> None: + """Raise the mapped error for a non-200 response, like subscriptionStatusError.""" + if h.status == 200: + return + if decimal(h.get("content-length")) != 0: + raise ProtocolError("error response with a body") + if h.status == 416: + v = h.get("content-range") or "" + if not v.startswith("bytes */"): + raise ProtocolError("416 without Content-Range") + decimal(v[len("bytes */") :]) + elif h.get("content-range") is not None: + raise ProtocolError("unexpected Content-Range") + raise status_error(h.status) diff --git a/clients/python/racer/src/racer/client.py b/clients/python/racer/src/racer/client.py new file mode 100644 index 0000000000..c2146395b5 --- /dev/null +++ b/clients/python/racer/src/racer/client.py @@ -0,0 +1,692 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Racer client over the node-local Unix socket. + +Request heads and response checks mirror pkg/racersdk (client.go, object.go) +and pkg/racersdk/internal/wire/wire.go. The host-read and gpu-read RPCs follow +designs/racer-gpudirect.md. +""" + +from __future__ import annotations + +import array +import contextlib +import os +import socket +import threading +from dataclasses import dataclass +from typing import Callable, Optional + +from . import _wire as wire +from .errors import ProtocolError, RacerError, StatusError, TruncatedError, UnsatisfiableRange +from .keys import Request + +OnPage = Optional[Callable[[int, int], None]] + +GPU = "gpu" +HOST = "host" +STREAM = "stream" + +# Statuses that mean "this dataplane does not support the mode" on a first +# attempt (designs/racer-gpudirect.md, Fallback). +FALLBACK_STATUSES = frozenset({400, 404, 405, 501}) + +DEFAULT_STAGING_BYTES = 4 * wire.PAGE_SIZE + +_SUPPORTED = "supported" +_UNSUPPORTED = "unsupported" + +# Mode support per socket path, kept until the process exits. +_mode_lock = threading.Lock() +_mode_state: dict[str, dict[str, str]] = {} + + +def _dest_device(dest) -> int | None: + """Return the CUDA device ordinal of a read destination, if it has one.""" + device = getattr(dest, "device", None) + if isinstance(device, int) and not isinstance(device, bool): + return device + index = getattr(device, "index", None) + return index if isinstance(index, int) else None + + +def socket_path_for(cache: str | None = None, socket_path: str | None = None) -> str: + """Resolve the client socket path. + + Order: explicit socket_path, then $RACER_SOCKET, then + /run/racer//client/socket (pkg/racersdk/client.go). + """ + if socket_path: + return socket_path + env = os.environ.get("RACER_SOCKET") + if env: + return env + if not cache: + raise RacerError("need a cache name, a socket path, or RACER_SOCKET") + if "/" in cache or cache in (".", ".."): + raise RacerError("invalid cache name") + return f"/run/racer/{cache}/client/socket" + + +def forget_modes(path: str | None = None) -> None: + """Clear remembered mode support (for tests, or after a dataplane upgrade).""" + with _mode_lock: + if path is None: + _mode_state.clear() + else: + _mode_state.pop(path, None) + + +@dataclass(frozen=True) +class Metadata: + etag: str + size: int + expires_at_ms: int + content_type: str | None = None + retain_until_ms: int | None = None + + +@dataclass(frozen=True) +class ReadResult: + metadata: Metadata + first: int + end: int + delivery: str + pages: int + + @property + def length(self) -> int: + return self.end - self.first + + +class _Conn: + def __init__(self, path: str, timeout: float | None) -> None: + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.settimeout(timeout) + try: + self.sock.connect(path) + except BaseException: + self.sock.close() + raise + self.buf = bytearray() + + def close(self) -> None: + self.sock.close() + + def send_head(self, head: bytes, fd: int | None = None) -> None: + if fd is None: + self.sock.sendall(head) + return + # The descriptor rides on the sendmsg that carries the first head byte. + anc = [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", [fd]).tobytes())] + sent = self.sock.sendmsg([head], anc) + if sent < len(head): + self.sock.sendall(head[sent:]) + + def send_credit(self, number: int, length: int) -> None: + # Racer may finish and close while a credit is in flight; the next read + # reports whether the transfer was cut short (object.go credit()). + with contextlib.suppress(BrokenPipeError, ConnectionResetError): + self.sock.sendall(wire.encode_credit(number, length)) + + def read_head(self) -> bytes: + while True: + i = self.buf.find(b"\r\n\r\n") + if i >= 0: + head = bytes(self.buf[: i + 4]) + del self.buf[: i + 4] + return head + if len(self.buf) > wire.MAX_HEAD_BYTES: + raise ProtocolError("response head too large") + chunk = self.sock.recv(65536) + if not chunk: + raise TruncatedError("connection closed before the response head") + self.buf += chunk + + def readinto(self, mv: memoryview) -> None: + n = min(len(self.buf), len(mv)) + if n: + mv[:n] = self.buf[:n] + del self.buf[:n] + while n < len(mv): + got = self.sock.recv_into(mv[n:]) + if got == 0: + raise TruncatedError("connection closed mid-transfer") + n += got + + def read_exact(self, size: int) -> bytes: + out = bytearray(size) + self.readinto(memoryview(out)) + return bytes(out) + + +def _response_metadata(h: wire.ResponseHead) -> Metadata | None: + etag = h.get("etag") or "" + try: + wire.validate_etag(etag) + expiry = wire.decimal(h.get("racer-expires-at")) + except RacerError: + return None + ct = h.get("racer-content-type") + if ct is not None and not wire.validate_content_type(ct): + return None + return Metadata(etag=etag, size=0, expires_at_ms=expiry, content_type=ct) + + +class Client: + """Reads Racer objects through the client socket. + + Each read uses its own connection, so one Client may be shared by threads. + """ + + def __init__( + self, + cache: str | None = None, + socket_path: str | None = None, + timeout: float | None = 60.0, + staging_bytes: int = DEFAULT_STAGING_BYTES, + pin_staging: bool = True, + ) -> None: + self.socket_path = socket_path_for(cache, socket_path) + self.timeout = timeout + if staging_bytes <= 0: + raise RacerError("staging_bytes must be positive") + self.staging_bytes = staging_bytes + self.pin_staging = pin_staging + self._pool: list = [] + self._pool_lock = threading.Lock() + + # Mode memory ----------------------------------------------------------- + + def mode_state(self, mode: str) -> str | None: + with _mode_lock: + return _mode_state.get(self.socket_path, {}).get(mode) + + def _set_mode(self, mode: str, state: str) -> None: + with _mode_lock: + _mode_state.setdefault(self.socket_path, {})[mode] = state + + # Staging pool ---------------------------------------------------------- + + def _take_staging(self, device: int | None = None): + from .host import HostBuffer + + with self._pool_lock: + if self._pool: + return self._pool.pop() + buf = HostBuffer(self.staging_bytes, "racer-staging") + if self.pin_staging: + buf.pin(0 if device is None else device) + return buf + + def _give_staging(self, buf) -> None: + with self._pool_lock: + self._pool.append(buf) + + def close(self) -> None: + with self._pool_lock: + pool, self._pool = self._pool, [] + for b in pool: + b.close() + + def __enter__(self) -> "Client": + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + # Heads ----------------------------------------------------------------- + + @staticmethod + def _context_headers(request: Request) -> str: + out = "" + if request.metadata: + wire.validate_opaque(request.metadata) + out += "Racer-Metadata: " + request.metadata + "\r\n" + if request.authorization: + wire.validate_opaque(request.authorization) + out += "Authorization: " + request.authorization + "\r\n" + return out + + @staticmethod + def _range_header(offset: int, length: int | None) -> str: + if offset < 0 or (length is not None and length < 0) or offset > wire.MAX_WIRE_INTEGER: + raise RacerError("invalid range") + if length and offset + length - 1 > wire.MAX_WIRE_INTEGER: + raise RacerError("invalid range") + if offset == 0 and not length: + return "" + out = f"Range: bytes={offset}-" + if length: + out += str(offset + length - 1) + return out + "\r\n" + + def _head_bytes(self, head: str) -> bytes: + raw = head.encode() + if len(raw) > wire.MAX_HEAD_BYTES: + raise RacerError("request head too large") + return raw + + def _connect(self) -> _Conn: + return _Conn(self.socket_path, self.timeout) + + # Stat ------------------------------------------------------------------ + + def stat(self, request: Request, if_match: str | None = None, retain_seconds: int | None = None) -> Metadata: + """HEAD an object. retain_seconds asks Racer to keep it warm.""" + head = f"HEAD {wire.OBJECT_PREFIX}{request.hex} HTTP/1.1\r\nHost: racer\r\n" + if if_match: + wire.validate_etag(if_match) + head += "If-Match: " + if_match + "\r\n" + if retain_seconds: + head += f"Cache-Control: max-age={int(retain_seconds)}\r\n" + head += self._context_headers(request) + "\r\n" + conn = self._connect() + try: + conn.send_head(self._head_bytes(head)) + h = wire.parse_response_head(conn.read_head()) + finally: + conn.close() + wire.check_error_status(h) + size = wire.decimal(h.get("content-length")) + if h.get("content-type") != "application/octet-stream": + raise ProtocolError("bad Content-Type") + m = _response_metadata(h) + if m is None or (if_match and m.etag != if_match): + raise ProtocolError("bad HEAD metadata") + retain = h.get("racer-retain-until") + return Metadata( + etag=m.etag, + size=size, + expires_at_ms=m.expires_at_ms, + content_type=m.content_type, + retain_until_ms=wire.decimal(retain) if retain is not None else None, + ) + + # Shared response checks ------------------------------------------------ + + @staticmethod + def _check_selection( + h: wire.ResponseHead, offset: int, length: int | None, if_match: str | None + ) -> tuple[Metadata, int, int]: + """Validate a 200 subscription-style head (wire.ParseSubscriptionResponse).""" + if not wire.connection_close(h) or h.get("content-range") is not None: + raise ProtocolError("bad response head") + if h.get("content-type") != "application/octet-stream": + raise ProtocolError("bad Content-Type") + size = wire.decimal(h.get("racer-object-length")) + first = wire.decimal(h.get("racer-range-start")) + end = wire.decimal(h.get("racer-range-end")) + m = _response_metadata(h) + if m is None or first > end or end > size or first != offset: + raise ProtocolError("bad response selection") + if not length and end != size: + raise ProtocolError("bad response selection") + if if_match and m.etag != if_match: + raise ProtocolError("response ETag does not match If-Match") + if length and end - first != length: + if end == size and length > end - first: + raise UnsatisfiableRange(416, "range extends past the object") + raise ProtocolError("bad response selection") + meta = Metadata(m.etag, size, m.expires_at_ms, m.content_type) + return meta, first, end + + # Stream mode ----------------------------------------------------------- + + def _stream( + self, + request: Request, + offset: int, + length: int | None, + if_match: str | None, + scratch: memoryview, + write: Callable[[int, memoryview], None], + on_page: OnPage, + max_length: int | None = None, + on_head: Callable[[], None] | None = None, + ) -> ReadResult: + head = ( + f"POST {wire.OBJECT_PREFIX}{request.hex} HTTP/1.1\r\nHost: racer\r\nContent-Length: 0\r\n" + f"Racer-Page-Credits: {wire.PAGE_CREDITS}\r\nRacer-Byte-Credits: {wire.BYTE_CREDITS}\r\n" + ) + head += self._range_header(offset, length) + if if_match: + wire.validate_etag(if_match) + head += "If-Match: " + if_match + "\r\n" + head += self._context_headers(request) + "\r\n" + if len(scratch) == 0: + raise RacerError("empty scratch buffer") + conn = self._connect() + try: + conn.send_head(self._head_bytes(head)) + h = wire.parse_response_head(conn.read_head()) + wire.check_error_status(h) + meta, first, end = self._check_selection(h, offset, length, if_match) + pages = wire.page_count(first, end) + body = wire.decimal(h.get("content-length")) + if body != end - first + wire.FRAME_SIZE * (pages + 1): + raise ProtocolError("bad Content-Length") + if max_length is not None and end - first > max_length: + # An earlier direct mode may have refused this read with 400 + # for the same reason; that refusal says nothing about support + # (designs/racer-gpudirect.md, Fallback), so do not confirm it. + raise RacerError("destination is smaller than the range") + if on_head: + on_head() + seq = wire.Sequence(first, end) + for _ in range(pages): + f = wire.decode_frame(conn.read_exact(wire.FRAME_SIZE)) + seq.accept(f) + if f.kind != wire.PAGE_FRAME: + raise ProtocolError("missing page frame") + pos = 0 + while pos < f.length: + n = min(len(scratch), f.length - pos) + conn.readinto(scratch[:n]) + write(f.offset + pos, scratch[:n]) + pos += n + conn.send_credit(f.number, f.length) + if on_page: + on_page(f.offset, f.length) + f = wire.decode_frame(conn.read_exact(wire.FRAME_SIZE)) + seq.accept(f) + if f.kind != wire.COMPLETE_FRAME: # pragma: no cover - accept() enforces it + raise ProtocolError("missing completion") + return ReadResult(meta, first, end, STREAM, pages) + finally: + conn.close() + + def read(self, request: Request, offset: int = 0, length: int | None = None, if_match: str | None = None) -> bytes: + """Read a range into new bytes using stream mode.""" + out = bytearray() + + def write(o: int, piece: memoryview) -> None: + out.extend(piece) + + self._stream(request, offset, length, if_match, memoryview(bytearray(1 << 20)), write, None) + return bytes(out) + + # RPC modes ------------------------------------------------------------- + + def _rpc( + self, + mode: str, + request: Request, + fd: int, + dest_offset: int, + dest_length: int, + allocation_size: int | None, + offset: int, + length: int | None, + if_match: str | None, + on_frame: Callable[[int, int], None] | None, + on_head: Callable[[], None] | None = None, + ) -> ReadResult: + if dest_offset < 0 or dest_length <= 0: + raise RacerError("invalid destination window") + head = f"POST {wire.OBJECT_PREFIX}{request.hex}/{mode}-read HTTP/1.1\r\nHost: racer\r\nContent-Length: 0\r\n" + head += self._range_header(offset, length) + if if_match: + wire.validate_etag(if_match) + head += "If-Match: " + if_match + "\r\n" + head += f"Racer-Destination-Offset: {dest_offset}\r\nRacer-Destination-Length: {dest_length}\r\n" + if allocation_size is not None: + head += f"Racer-Allocation-Size: {allocation_size}\r\n" + head += self._context_headers(request) + "\r\n" + conn = self._connect() + try: + conn.send_head(self._head_bytes(head), fd) + h = wire.parse_response_head(conn.read_head()) + wire.check_error_status(h) + if h.get("racer-delivery") != mode: + raise ProtocolError("bad Racer-Delivery") + meta, first, end = self._check_selection(h, offset, length, if_match) + if end - first > dest_length: + raise ProtocolError("range is larger than the destination") + if on_head: + on_head() + pages = wire.page_count(first, end) + if wire.decimal(h.get("content-length")) != wire.FRAME_SIZE * (pages + 1): + raise ProtocolError("bad Content-Length") + seq = wire.Sequence(first, end) + for _ in range(pages): + f = wire.decode_frame(conn.read_exact(wire.FRAME_SIZE)) + seq.accept(f) + if f.kind != wire.PAGE_FRAME: + raise ProtocolError("missing page frame") + if on_frame: + on_frame(f.offset, f.length) + f = wire.decode_frame(conn.read_exact(wire.FRAME_SIZE)) + seq.accept(f) + return ReadResult(meta, first, end, mode, pages) + finally: + conn.close() + + def host_read( + self, + request: Request, + fd: int, + dest_offset: int, + dest_length: int, + *, + offset: int = 0, + length: int | None = None, + if_match: str | None = None, + on_page: OnPage = None, + ) -> ReadResult: + """One host-read RPC with no fallback. fd is a regular file, normally a memfd.""" + return self._rpc(HOST, request, fd, dest_offset, dest_length, None, offset, length, if_match, on_page) + + def gpu_read( + self, + request: Request, + fd: int, + allocation_size: int, + dest_offset: int, + dest_length: int, + *, + offset: int = 0, + length: int | None = None, + if_match: str | None = None, + on_page: OnPage = None, + ) -> ReadResult: + """One gpu-read RPC with no fallback. fd comes from cuMemExportToShareableHandle.""" + if dest_offset + dest_length > allocation_size: + raise RacerError("destination window is outside the allocation") + return self._rpc(GPU, request, fd, dest_offset, dest_length, allocation_size, offset, length, if_match, on_page) + + # Fallback driver ------------------------------------------------------- + + def _run_modes(self, attempts: list[tuple[str, Callable[[Callable[[], None]], ReadResult]]]) -> ReadResult: + """Try modes in order. + + A mode with no remembered state that answers 400/404/405/501 is + skipped. 501 marks it unsupported at once. 400/404/405 mark it + unsupported only once a later mode gets a 200 head, so a missing + object (404 everywhere) does not disable a mode for good. + """ + pending: list[str] = [] + last: StatusError | None = None + + def understood() -> None: + for m in pending: + self._set_mode(m, _UNSUPPORTED) + pending.clear() + + for mode, run in attempts: + state = self.mode_state(mode) if mode != STREAM else _SUPPORTED + if state == _UNSUPPORTED: + continue + try: + result = run(understood) + except _Skip: + continue + except StatusError as exc: + if mode == STREAM or state is not None or exc.status not in FALLBACK_STATUSES: + raise + if exc.status == 501: + self._set_mode(mode, _UNSUPPORTED) + else: + pending.append(mode) + last = exc + continue + if mode != STREAM: + self._set_mode(mode, _SUPPORTED) + return result + if last is not None: # pragma: no cover - stream always runs last + raise last + raise RacerError("no read mode available") + + # High level: host ------------------------------------------------------ + + def read_into_host( + self, + request: Request, + dest, + dest_offset: int = 0, + *, + offset: int = 0, + length: int | None = None, + if_match: str | None = None, + on_page: OnPage = None, + ) -> ReadResult: + """Read a range into host memory at dest_offset. + + dest is a HostBuffer (host-read, then stream) or any writable buffer + (stream only). Object offset o lands at dest_offset + (o - offset). + """ + view = memoryview(dest.view if hasattr(dest, "view") else dest).cast("B") + window = len(view) - dest_offset + if dest_offset < 0 or window <= 0: + raise RacerError("invalid destination window") + if length and length > window: + raise RacerError("destination is smaller than the range") + attempts = [] + if hasattr(dest, "fileno"): + attempts.append( + ( + HOST, + lambda ok: self._rpc( + HOST, request, dest.fileno(), dest_offset, window, None, offset, length, if_match, on_page, ok + ), + ) + ) + + def stream(ok): + def write(o: int, piece: memoryview) -> None: + at = dest_offset + o - offset + view[at : at + len(piece)] = piece + + scratch = memoryview(bytearray(min(window, 1 << 20))) + return self._stream(request, offset, length, if_match, scratch, write, on_page, window, ok) + + attempts.append((STREAM, stream)) + try: + return self._run_modes(attempts) + finally: + view.release() + + # High level: GPU ------------------------------------------------------- + + def read_into_gpu( + self, + request: Request, + dest, + dest_offset: int = 0, + *, + offset: int = 0, + length: int | None = None, + if_match: str | None = None, + on_page: OnPage = None, + ) -> ReadResult: + """Read a range into GPU memory at dest_offset. + + dest is a racer.cuda.GpuAllocation, a racer.tensors.TensorTarget, or + any object with ``size``, ``copy_from_host(offset, data)`` and + optionally ``export_fd()``. Modes: gpu-read, then host-read into a + pinned staging memfd plus a host-to-device copy, then stream into the + staging buffer plus a copy. on_page(object_offset, length) runs once a + page is in GPU memory. + """ + window = dest.size - dest_offset + if dest_offset < 0 or window <= 0: + raise RacerError("invalid destination window") + if length and length > window: + raise RacerError("destination is smaller than the range") + + attempts: list = [] + if hasattr(dest, "export_fd"): + + def gpu(ok): + try: + fd = dest.export_fd() + except NotImplementedError: + raise _Skip() from None + try: + return self._rpc( + GPU, request, fd, dest_offset, window, dest.size, offset, length, if_match, on_page, ok + ) + finally: + os.close(fd) + + attempts.append((GPU, gpu)) + attempts.append((HOST, lambda ok: self._gpu_via_host(request, dest, dest_offset, window, offset, length, if_match, on_page, ok))) + attempts.append((STREAM, lambda ok: self._gpu_via_stream(request, dest, dest_offset, window, offset, length, if_match, on_page, ok))) + + return self._run_modes(attempts) + + def _gpu_via_host(self, request, dest, dest_offset, window, offset, length, if_match, on_page, ok) -> ReadResult: + staging = self._take_staging(_dest_device(dest)) + try: + chunk = staging.size + if length: + total = length + else: + # Resolve the size once so every chunk reads the same version. + meta = self.stat(request, if_match=if_match) + if offset > meta.size: + raise UnsatisfiableRange(416) + total = meta.size - offset + if_match = meta.etag + ok() + if total > window: + raise RacerError("destination is smaller than the range") + if total == 0: + return self._rpc(HOST, request, staging.fileno(), 0, chunk, None, offset, None, if_match, None, ok) + first_result = None + pos = 0 + pages = 0 + while pos < total: + n = min(chunk, total - pos) + base = offset + pos + + def frame(o: int, size: int, base=base) -> None: + dest.copy_from_host(dest_offset + (o - offset), staging.view[o - base : o - base + size]) + if on_page: + on_page(o, size) + + r = self._rpc(HOST, request, staging.fileno(), 0, n, None, base, n, if_match, frame, ok) + if first_result is None: + first_result = r + if_match = r.metadata.etag + pages += r.pages + pos += n + assert first_result is not None + return ReadResult(first_result.metadata, offset, offset + total, HOST, pages) + finally: + self._give_staging(staging) + + def _gpu_via_stream(self, request, dest, dest_offset, window, offset, length, if_match, on_page, ok) -> ReadResult: + staging = self._take_staging(_dest_device(dest)) + try: + + def write(o: int, piece: memoryview) -> None: + dest.copy_from_host(dest_offset + (o - offset), piece) + + return self._stream(request, offset, length, if_match, staging.view, write, on_page, window, ok) + finally: + self._give_staging(staging) + + +class _Skip(Exception): + """The destination cannot use this mode. Nothing is remembered.""" diff --git a/clients/python/racer/src/racer/cuda.py b/clients/python/racer/src/racer/cuda.py new file mode 100644 index 0000000000..0ab7e3e390 --- /dev/null +++ b/clients/python/racer/src/racer/cuda.py @@ -0,0 +1,430 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Minimal ctypes bindings to the CUDA driver API (libcuda.so.1). + +Only what Racer needs: exportable device allocations through the virtual +memory management (VMM) API, host-to-device copies, and host registration. + +Struct layouts and enum values come from cuda.h in the CUDA toolkit +(https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__TYPES.html and +https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__VA.html): + + typedef struct CUmemLocation_st { CUmemLocationType type; int id; } CUmemLocation; + typedef struct CUmemAllocationProp_st { + CUmemAllocationType type; + CUmemAllocationHandleType requestedHandleTypes; + CUmemLocation location; + void *win32HandleMetaData; + struct { + unsigned char compressionType; + unsigned char gpuDirectRDMACapable; + unsigned short usage; + unsigned char reserved[4]; + } allocFlags; + } CUmemAllocationProp; + typedef struct CUmemAccessDesc_st { CUmemLocation location; CUmemAccess_flags flags; } CUmemAccessDesc; + + CU_MEM_ALLOCATION_TYPE_PINNED = 0x1 + CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR = 0x1 + CU_MEM_LOCATION_TYPE_DEVICE = 0x1 + CU_MEM_ACCESS_FLAGS_PROT_READWRITE = 0x3 + CU_MEM_ALLOC_GRANULARITY_MINIMUM = 0x0 + +Versioned symbols follow the cuda.h #defines: cuMemcpyHtoD -> cuMemcpyHtoD_v2, +cuMemHostRegister -> cuMemHostRegister_v2, cuDevicePrimaryCtxRelease -> +cuDevicePrimaryCtxRelease_v2. +""" + +from __future__ import annotations + +import contextlib +import ctypes +import logging +import os +import threading + +from .errors import RacerError + +log = logging.getLogger(__name__) + +CU_MEM_ALLOCATION_TYPE_PINNED = 0x1 +CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR = 0x1 +CU_MEM_LOCATION_TYPE_DEVICE = 0x1 +CU_MEM_ACCESS_FLAGS_PROT_READWRITE = 0x3 +CU_MEM_ALLOC_GRANULARITY_MINIMUM = 0x0 +CU_MEMHOSTREGISTER_PORTABLE = 0x1 + +CUdeviceptr = ctypes.c_ulonglong +CUmemGenericAllocationHandle = ctypes.c_ulonglong + + +class CudaUnavailable(RacerError): + """libcuda is missing or the driver cannot be initialized.""" + + +class CudaError(RacerError): + def __init__(self, func: str, code: int) -> None: + super().__init__(f"{func} failed with CUresult {code}") + self.func = func + self.code = code + + +class CUmemLocation(ctypes.Structure): + _fields_ = [("type", ctypes.c_int), ("id", ctypes.c_int)] + + +class _AllocFlags(ctypes.Structure): + _fields_ = [ + ("compressionType", ctypes.c_ubyte), + ("gpuDirectRDMACapable", ctypes.c_ubyte), + ("usage", ctypes.c_ushort), + ("reserved", ctypes.c_ubyte * 4), + ] + + +class CUmemAllocationProp(ctypes.Structure): + _fields_ = [ + ("type", ctypes.c_int), + ("requestedHandleTypes", ctypes.c_int), + ("location", CUmemLocation), + ("win32HandleMetaData", ctypes.c_void_p), + ("allocFlags", _AllocFlags), + ] + + +class CUmemAccessDesc(ctypes.Structure): + _fields_ = [("location", CUmemLocation), ("flags", ctypes.c_int)] + + +_lock = threading.Lock() +_lib = None +_contexts: dict[int, ctypes.c_void_p] = {} + + +def _load(): + global _lib + with _lock: + if _lib is not None: + return _lib + name = os.environ.get("RACER_LIBCUDA", "libcuda.so.1") + try: + lib = ctypes.CDLL(name) + except OSError as exc: + raise CudaUnavailable(f"cannot load {name}: {exc}") from exc + sig = { + "cuInit": [ctypes.c_uint], + "cuDeviceGet": [ctypes.POINTER(ctypes.c_int), ctypes.c_int], + "cuCtxGetCurrent": [ctypes.POINTER(ctypes.c_void_p)], + "cuCtxSetCurrent": [ctypes.c_void_p], + "cuDevicePrimaryCtxRetain": [ctypes.POINTER(ctypes.c_void_p), ctypes.c_int], + "cuMemGetAllocationGranularity": [ + ctypes.POINTER(ctypes.c_size_t), + ctypes.POINTER(CUmemAllocationProp), + ctypes.c_int, + ], + "cuMemCreate": [ + ctypes.POINTER(CUmemGenericAllocationHandle), + ctypes.c_size_t, + ctypes.POINTER(CUmemAllocationProp), + ctypes.c_ulonglong, + ], + "cuMemRelease": [CUmemGenericAllocationHandle], + "cuMemAddressReserve": [ + ctypes.POINTER(CUdeviceptr), + ctypes.c_size_t, + ctypes.c_size_t, + CUdeviceptr, + ctypes.c_ulonglong, + ], + "cuMemAddressFree": [CUdeviceptr, ctypes.c_size_t], + "cuMemMap": [CUdeviceptr, ctypes.c_size_t, ctypes.c_size_t, CUmemGenericAllocationHandle, ctypes.c_ulonglong], + "cuMemUnmap": [CUdeviceptr, ctypes.c_size_t], + "cuMemSetAccess": [CUdeviceptr, ctypes.c_size_t, ctypes.POINTER(CUmemAccessDesc), ctypes.c_size_t], + "cuMemExportToShareableHandle": [ + ctypes.c_void_p, + CUmemGenericAllocationHandle, + ctypes.c_int, + ctypes.c_ulonglong, + ], + "cuMemcpyHtoD_v2": [CUdeviceptr, ctypes.c_void_p, ctypes.c_size_t], + "cuMemcpyDtoH_v2": [ctypes.c_void_p, CUdeviceptr, ctypes.c_size_t], + "cuMemHostRegister_v2": [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint], + "cuMemHostUnregister": [ctypes.c_void_p], + } + for fn, args in sig.items(): + f = getattr(lib, fn, None) + if f is None: + raise CudaUnavailable(f"libcuda has no {fn}") + f.argtypes = args + f.restype = ctypes.c_int + rc = lib.cuInit(0) + if rc != 0: + raise CudaUnavailable(f"cuInit failed with CUresult {rc}") + _lib = lib + return lib + + +def _check(func: str, rc: int) -> None: + if rc != 0: + raise CudaError(func, rc) + + +def available() -> bool: + try: + _load() + except CudaUnavailable: + return False + return True + + +def ensure_context(device: int = 0) -> None: + """Make sure a context is current; retain the primary context if not. + + torch uses the primary context too, so both share one context. + """ + lib = _load() + cur = ctypes.c_void_p() + _check("cuCtxGetCurrent", lib.cuCtxGetCurrent(ctypes.byref(cur))) + if cur.value: + return + _check("cuCtxSetCurrent", lib.cuCtxSetCurrent(_primary_context(device))) + + +def _primary_context(device: int) -> ctypes.c_void_p: + """Return the device's primary context, retained once per process. + + The retain is never released: torch shares the same primary context and + the cached handle stays valid for the life of the process. + """ + lib = _load() + with _lock: + ctx = _contexts.get(device) + if ctx is None: + dev = ctypes.c_int() + _check("cuDeviceGet", lib.cuDeviceGet(ctypes.byref(dev), device)) + ctx = ctypes.c_void_p() + _check("cuDevicePrimaryCtxRetain", lib.cuDevicePrimaryCtxRetain(ctypes.byref(ctx), dev.value)) + _contexts[device] = ctx + return ctx + + +@contextlib.contextmanager +def bound_context(device: int): + """Make ``device``'s primary context current, restoring the previous one after. + + Worker threads do not inherit a context from the thread that created an + allocation, and an unrelated current context (for example another + device's) is wrong for driver calls on this device's memory, so callers + bind explicitly instead of relying on thread history. Uses + cuCtxGetCurrent/cuCtxSetCurrent + (https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html). + """ + lib = _load() + ctx = _primary_context(device) + prev = ctypes.c_void_p() + _check("cuCtxGetCurrent", lib.cuCtxGetCurrent(ctypes.byref(prev))) + if prev.value == ctx.value: + yield + return + _check("cuCtxSetCurrent", lib.cuCtxSetCurrent(ctx)) + try: + yield + finally: + rc = lib.cuCtxSetCurrent(prev) + if rc != 0: + log.warning("racer: restoring the CUDA context failed with CUresult %d", rc) + + +def _address(buf) -> tuple[int, object]: + """Return the address of a buffer and an object that keeps it valid.""" + mv = memoryview(buf).cast("B") + if mv.readonly: + holder = ctypes.create_string_buffer(mv.tobytes(), len(mv)) + return ctypes.addressof(holder), holder + arr = (ctypes.c_char * len(mv)).from_buffer(mv) + return ctypes.addressof(arr), arr + + +def _allocation_prop(device: int) -> CUmemAllocationProp: + prop = CUmemAllocationProp() + prop.type = CU_MEM_ALLOCATION_TYPE_PINNED + prop.requestedHandleTypes = CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR + prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE + prop.location.id = device + return prop + + +def granularity(device: int = 0) -> int: + lib = _load() + ensure_context(device) + g = ctypes.c_size_t() + prop = _allocation_prop(device) + _check( + "cuMemGetAllocationGranularity", + lib.cuMemGetAllocationGranularity(ctypes.byref(g), ctypes.byref(prop), CU_MEM_ALLOC_GRANULARITY_MINIMUM), + ) + return g.value + + +class GpuAllocation: + """Exportable device memory, mapped read-write in this process. + + ``size`` is the size passed to cuMemCreate, rounded up to the allocation + granularity. It is what gpu-read sends as Racer-Allocation-Size. + + The object implements ``__cuda_array_interface__`` (a flat uint8 array), + so torch, CuPy and Numba can wrap it without a copy. Any wrapper must keep + this object alive; ``racer.torch.as_tensor`` does. + """ + + def __init__(self, size: int, device: int = 0) -> None: + if size <= 0: + raise RacerError("size must be positive") + lib = _load() + g = granularity(device) + self.device = device + self.requested = size + self.size = (size + g - 1) // g * g + self.ptr = 0 + self._handle = None + self._reserved = False + self._mapped = False + prop = _allocation_prop(device) + handle = CUmemGenericAllocationHandle() + with bound_context(device): + try: + _check("cuMemCreate", lib.cuMemCreate(ctypes.byref(handle), self.size, ctypes.byref(prop), 0)) + self._handle = handle.value + ptr = CUdeviceptr() + _check("cuMemAddressReserve", lib.cuMemAddressReserve(ctypes.byref(ptr), self.size, g, 0, 0)) + self.ptr = ptr.value + self._reserved = True + _check("cuMemMap", lib.cuMemMap(self.ptr, self.size, 0, self._handle, 0)) + self._mapped = True + desc = CUmemAccessDesc() + desc.location.type = CU_MEM_LOCATION_TYPE_DEVICE + desc.location.id = device + desc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE + _check("cuMemSetAccess", lib.cuMemSetAccess(self.ptr, self.size, ctypes.byref(desc), 1)) + except BaseException: + self.close() + raise + + def export_fd(self) -> int: + """Return a new POSIX file descriptor for the allocation. The caller closes it.""" + lib = _load() + fd = ctypes.c_int(-1) + _check( + "cuMemExportToShareableHandle", + lib.cuMemExportToShareableHandle(ctypes.byref(fd), self._handle, CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR, 0), + ) + return fd.value + + def copy_from_host(self, offset: int, data) -> None: + mv = memoryview(data).cast("B") + if offset < 0 or offset + len(mv) > self.size: + raise RacerError("copy outside the allocation") + if not mv: + return + addr, keep = _address(mv) + try: + with bound_context(self.device): + _check("cuMemcpyHtoD_v2", _load().cuMemcpyHtoD_v2(self.ptr + offset, addr, len(mv))) + finally: + del keep + + def copy_to_host(self, offset: int, out) -> None: + mv = memoryview(out).cast("B") + if offset < 0 or offset + len(mv) > self.size: + raise RacerError("copy outside the allocation") + addr, keep = _address(mv) + try: + with bound_context(self.device): + _check("cuMemcpyDtoH_v2", _load().cuMemcpyDtoH_v2(addr, self.ptr + offset, len(mv))) + finally: + del keep + + @property + def __cuda_array_interface__(self) -> dict: + return { + "shape": (self.size,), + "typestr": "|u1", + "data": (self.ptr, False), + "strides": None, + "version": 3, + } + + def close(self) -> None: + """Unmap and release. On failure the resource is kept (leaked), never freed early.""" + if _lib is None: + return + lib = _lib + try: + if self._mapped: + _check("cuMemUnmap", lib.cuMemUnmap(self.ptr, self.size)) + self._mapped = False + if self._reserved: + _check("cuMemAddressFree", lib.cuMemAddressFree(self.ptr, self.size)) + self._reserved = False + if self._handle is not None: + _check("cuMemRelease", lib.cuMemRelease(self._handle)) + self._handle = None + except CudaError as exc: + log.warning("racer: leaking GPU allocation: %s", exc) + self._mapped = self._reserved = False + self._handle = None + + def __enter__(self) -> "GpuAllocation": + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + def __del__(self) -> None: + try: + self.close() + except Exception: + # Finalizers must never raise, and during interpreter shutdown the + # CUDA library or module globals may already be gone. Callers that + # need to observe free errors should call close() explicitly. + return + + +class HostRegistration: + def __init__(self, addr: int, keep: object) -> None: + self._addr = addr + self._keep = keep + + def close(self) -> None: + if self._addr: + rc = _load().cuMemHostUnregister(self._addr) + if rc != 0: + log.warning("racer: cuMemHostUnregister failed with CUresult %d", rc) + self._addr = 0 + self._keep = None + + +def host_register(buf, device: int = 0) -> HostRegistration: + """Page-lock a writable host buffer with cuMemHostRegister. + + The registration is made in ``device``'s primary context (PORTABLE makes + it usable from every context); pass the device the buffer will feed. + """ + lib = _load() + mv = memoryview(buf).cast("B") + arr = None + try: + if mv.readonly: + raise RacerError("buffer must be writable") + n = len(mv) + arr = (ctypes.c_char * n).from_buffer(mv) + addr = ctypes.addressof(arr) + with bound_context(device): + _check("cuMemHostRegister_v2", lib.cuMemHostRegister_v2(addr, n, CU_MEMHOSTREGISTER_PORTABLE)) + finally: + # Drop the ctypes export and the view even on failure, so a traceback + # that references this frame cannot keep the owner's mapping exported. + # The ctypes array holds an export of mv, so it must go before release. + del arr + mv.release() + return HostRegistration(addr, None) diff --git a/clients/python/racer/src/racer/errors.py b/clients/python/racer/src/racer/errors.py new file mode 100644 index 0000000000..f567250f09 --- /dev/null +++ b/clients/python/racer/src/racer/errors.py @@ -0,0 +1,84 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Errors raised by the Racer client. + +Status mapping follows wire.StatusError in pkg/racersdk/internal/wire/wire.go. +""" + +from __future__ import annotations + + +class RacerError(Exception): + """Base class for all Racer client errors.""" + + +class ProtocolError(RacerError): + """The server sent something the protocol does not allow.""" + + +class TruncatedError(RacerError): + """The connection closed before the completion frame.""" + + +class StatusError(RacerError): + """The server answered with a non-200 status.""" + + status = 0 + + def __init__(self, status: int, message: str | None = None) -> None: + super().__init__(message or f"racer: status {status}") + self.status = status + + +class InvalidArgument(StatusError): + pass + + +class Unauthorized(StatusError): + pass + + +class Forbidden(StatusError): + pass + + +class NotFound(StatusError): + pass + + +class VersionUnavailable(StatusError): + pass + + +class UnsatisfiableRange(StatusError): + pass + + +class HeaderLimit(StatusError): + pass + + +class NotImplementedStatus(StatusError): + pass + + +class Unavailable(StatusError): + pass + + +_BY_STATUS = { + 400: InvalidArgument, + 401: Unauthorized, + 403: Forbidden, + 404: NotFound, + 405: InvalidArgument, + 412: VersionUnavailable, + 416: UnsatisfiableRange, + 431: HeaderLimit, + 501: NotImplementedStatus, + 503: Unavailable, +} + + +def status_error(status: int) -> StatusError: + return _BY_STATUS.get(status, StatusError)(status) diff --git a/clients/python/racer/src/racer/host.py b/clients/python/racer/src/racer/host.py new file mode 100644 index 0000000000..90d1fe49b7 --- /dev/null +++ b/clients/python/racer/src/racer/host.py @@ -0,0 +1,73 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Host memory backed by a memfd, the destination for host-read.""" + +from __future__ import annotations + +import mmap +import os + +from .errors import RacerError + + +class HostBuffer: + """A memfd of a fixed size, mapped read-write into this process. + + The dataplane writes into the file with positional writes, so the memfd is + not sealed. ``view`` is a writable memoryview over the mapping, so the + buffer also works as a stream-mode destination. Call ``pin()`` to register + the mapping with CUDA for faster host-to-device copies. + """ + + def __init__(self, size: int, name: str = "racer-host") -> None: + if size <= 0: + raise RacerError("size must be positive") + self.size = size + self._fd = os.memfd_create(name, os.MFD_CLOEXEC) + try: + os.ftruncate(self._fd, size) + self._map = mmap.mmap(self._fd, size, mmap.MAP_SHARED, mmap.PROT_READ | mmap.PROT_WRITE) + except BaseException: + os.close(self._fd) + raise + self.view = memoryview(self._map) + self._pinned = None + + def fileno(self) -> int: + if self._fd < 0: + raise RacerError("buffer closed") + return self._fd + + @property + def pinned(self) -> bool: + return self._pinned is not None + + def pin(self, device: int = 0) -> bool: + """Register the mapping with CUDA in ``device``'s context. Returns False when CUDA is missing.""" + if self._pinned is not None: + return True + from . import cuda + + try: + self._pinned = cuda.host_register(self.view, device) + except cuda.CudaUnavailable: + return False + return True + + def close(self) -> None: + if self._pinned is not None: + self._pinned.close() + self._pinned = None + if self._fd >= 0: + fd, self._fd = self._fd, -1 + try: + self.view.release() + self._map.close() + finally: + os.close(fd) + + def __enter__(self) -> "HostBuffer": + return self + + def __exit__(self, *exc: object) -> None: + self.close() diff --git a/clients/python/racer/src/racer/keys.py b/clients/python/racer/src/racer/keys.py new file mode 100644 index 0000000000..727ac48cb5 --- /dev/null +++ b/clients/python/racer/src/racer/keys.py @@ -0,0 +1,88 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Object keys for racer-object (S3) objects. + +Replicates NewRequest in internal/racerobject/racerobject.go: the key is the +SHA-256 of the canonical schema-1 JSON produced by Go's encoding/json, and +that JSON is sent as Racer-Metadata. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass + +from .errors import RacerError + +_BUCKET = re.compile(r"[A-Za-z0-9._-]{1,255}") + + +@dataclass(frozen=True) +class Request: + """One object to read: its 32-byte key plus opaque origin context.""" + + key: bytes + metadata: str | None = None + authorization: str | None = None + + def __post_init__(self) -> None: + if len(self.key) != 32: + raise RacerError("key must be 32 bytes") + + @property + def hex(self) -> str: + return self.key.hex() + + @classmethod + def from_hex(cls, key: str, metadata: str | None = None, authorization: str | None = None) -> "Request": + if len(key) != 64 or any(c not in "0123456789abcdef" for c in key): + raise RacerError("key must be 64 lower-case hex digits") + return cls(bytes.fromhex(key), metadata, authorization) + + +def _go_json_string(s: str) -> str: + # Go escapes <, >, & and U+2028, U+2029 by default (json.Marshal HTMLEscape), + # and writes other non-ASCII as raw UTF-8. Control characters match Python's + # short escapes (\b \f \n \r \t) and \u00XX otherwise. + out = json.dumps(s, ensure_ascii=False) + return ( + out.replace("<", "\\u003c") + .replace(">", "\\u003e") + .replace("&", "\\u0026") + .replace("\u2028", "\\u2028") + .replace("\u2029", "\\u2029") + ) + + +def canonical_json(bucket: str, key: str, version_id: str | None = None) -> str: + """Return the canonical schema-1 JSON, validating like validObject.""" + version_id = version_id or "" + if bucket in (".", "..") or not _BUCKET.fullmatch(bucket): + raise RacerError("invalid bucket") + for s in (key, version_id): + try: + s.encode("utf-8") + except UnicodeEncodeError as exc: + raise RacerError("invalid UTF-8") from exc + if not key or len(key.encode()) > 1024 or len(version_id.encode()) > 1024: + raise RacerError("invalid key or version") + out = '{"schema":1,"bucket":' + _go_json_string(bucket) + ',"key":' + _go_json_string(key) + if version_id: + out += ',"versionId":' + _go_json_string(version_id) + out += "}" + if len(out.encode()) > 8 << 10: + raise RacerError("object identity too large") + return out + + +def object_request(bucket: str, key: str, version_id: str | None = None) -> Request: + """Build the Request racer-object uses for an S3 object.""" + canonical = canonical_json(bucket, key, version_id) + return Request(hashlib.sha256(canonical.encode()).digest(), canonical) + + +def object_key(bucket: str, key: str, version_id: str | None = None) -> str: + """Return the 64-hex-digit Racer key for an S3 object.""" + return object_request(bucket, key, version_id).hex diff --git a/clients/python/racer/src/racer/tensors.py b/clients/python/racer/src/racer/tensors.py new file mode 100644 index 0000000000..be155ed002 --- /dev/null +++ b/clients/python/racer/src/racer/tensors.py @@ -0,0 +1,59 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""torch helpers. torch is imported only when these functions run. + +Lifetime rule: a tensor made by ``as_tensor`` holds a reference to the +GpuAllocation it views, so the allocation lives at least as long as the +tensor (and any view of it). Closing the allocation by hand while tensors +still use it is a bug. +""" + +from __future__ import annotations + + +def _torch(): + import torch # noqa: PLC0415 - lazy on purpose + + return torch + + +def as_tensor(allocation, device: int | None = None): + """Wrap a GpuAllocation as a flat uint8 CUDA tensor without copying. + + This uses ``torch.as_tensor`` on the object's ``__cuda_array_interface__``. + torch keeps a reference to the exporting object for as long as the + tensor's storage lives, which gives the lifetime rule above. + """ + torch = _torch() + dev = torch.device("cuda", allocation.device if device is None else device) + return torch.as_tensor(allocation, device=dev) + + +class TensorTarget: + """A GPU destination backed by an existing torch CUDA uint8 tensor. + + It cannot be exported, so reads use host or stream mode and copy with + ``Tensor.copy_``. + """ + + def __init__(self, tensor) -> None: + torch = _torch() + if tensor.dtype != torch.uint8 or tensor.dim() != 1 or not tensor.is_contiguous(): + raise ValueError("tensor must be a contiguous 1-D uint8 tensor") + self.tensor = tensor + self.size = tensor.numel() + # CUDA ordinal of the tensor, so host and stream staging pins memory + # on this GPU rather than device 0. + index = getattr(tensor.device, "index", None) + self.device = index if isinstance(index, int) else None + + def export_fd(self) -> int: + raise NotImplementedError("torch memory cannot be exported") + + def copy_from_host(self, offset: int, data) -> None: + torch = _torch() + mv = memoryview(data).cast("B") + if not mv: + return + src = torch.frombuffer(mv, dtype=torch.uint8) + self.tensor[offset : offset + len(mv)].copy_(src) diff --git a/clients/python/racer/tests/conftest.py b/clients/python/racer/tests/conftest.py new file mode 100644 index 0000000000..4478c36e02 --- /dev/null +++ b/clients/python/racer/tests/conftest.py @@ -0,0 +1,67 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +import os +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(__file__)) + +from fakeserver import FakeRacer # noqa: E402 + +import racer # noqa: E402 +import racer._wire # noqa: E402 + +SMALL_PAGE = 4096 + + +@pytest.fixture +def small_pages(monkeypatch): + monkeypatch.setattr(racer._wire, "PAGE_SIZE", SMALL_PAGE) + return SMALL_PAGE + + +@pytest.fixture +def server(tmp_path, small_pages): + s = FakeRacer(str(tmp_path / "s")) + racer.forget_modes() + yield s + s.close() + racer.forget_modes() + + +class FakeGpu: + """A GPU destination backed by a memfd, so the fake server can write it.""" + + def __init__(self, size, exportable=True): + self.size = size + self.device = 0 + self.fd = os.memfd_create("fake-gpu") + os.ftruncate(self.fd, size) + self.copies = 0 + if not exportable: + self.export_fd = self._no_export + + def _no_export(self): + raise NotImplementedError + + def export_fd(self): + return os.dup(self.fd) + + def copy_from_host(self, offset, data): + self.copies += 1 + os.pwrite(self.fd, bytes(data), offset) + + def read(self, offset, n): + return os.pread(self.fd, n, offset) + + def close(self): + os.close(self.fd) + + +@pytest.fixture +def gpu_writer(): + def write(fd, offset, data): + os.pwrite(fd, data, offset) + + return write diff --git a/clients/python/racer/tests/fakeserver.py b/clients/python/racer/tests/fakeserver.py new file mode 100644 index 0000000000..90e631f294 --- /dev/null +++ b/clients/python/racer/tests/fakeserver.py @@ -0,0 +1,284 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""A small fake Racer client-socket server for tests.""" + +from __future__ import annotations + +import array +import fcntl +import os +import socket +import stat +import threading +from dataclasses import dataclass, field + +import racer +import racer._wire as wire + + +@dataclass +class Obj: + data: bytes + etag: str = '"v1"' + content_type: str | None = None + expires_ms: int = 1_700_000_000_000 + + +@dataclass +class Seen: + method: str + target: str + headers: dict + fds: int + fd_kind: str | None = None + credits: list = field(default_factory=list) + + +class FakeRacer: + """Serves HEAD, stream Get, host-read and gpu-read on a Unix socket. + + old=True mimics a dataplane that predates host-read/gpu-read: the new + paths fail key parsing and get 400. gpu_status is the answer to gpu-read + (501 by default, the "no usable GPU" case). gpu_write, when set, is + called as gpu_write(fd, dest_offset, bytes) to emulate the GPU write. + """ + + def __init__(self, path: str) -> None: + self.path = path + self.objects: dict[str, Obj] = {} + self.old = False + self.gpu_status = 501 + self.gpu_write = None + self.host_status: int | None = None + self.bad_frame = False + self.truncate = False + self.seen: list[Seen] = [] + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.bind(path) + self.sock.listen(16) + self._stop = False + self._thread = threading.Thread(target=self._serve, daemon=True) + self._thread.start() + + def close(self) -> None: + self._stop = True + try: + c = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + c.connect(self.path) + c.close() + except OSError: + # The connect only wakes the accept loop; if it fails, the + # server is already gone or stopping, so there is nothing to do. + pass + self._thread.join(5) + self.sock.close() + + def _serve(self) -> None: + while not self._stop: + conn, _ = self.sock.accept() + if self._stop: + conn.close() + return + threading.Thread(target=self._handle, args=(conn,), daemon=True).start() + + # ------------------------------------------------------------------ + + def _recv_head(self, conn): + buf = b"" + fds: list[int] = [] + while b"\r\n\r\n" not in buf: + data, anc, flags, _ = conn.recvmsg(65536, socket.CMSG_SPACE(4 * 8), socket.MSG_CMSG_CLOEXEC) + for level, typ, cdata in anc: + if level == socket.SOL_SOCKET and typ == socket.SCM_RIGHTS: + a = array.array("i") + a.frombytes(cdata[: len(cdata) - len(cdata) % 4]) + fds.extend(a) + if not data: + return None, None, fds + buf += data + head, _, rest = buf.partition(b"\r\n\r\n") + return head.decode("utf-8", "surrogateescape"), rest, fds + + @staticmethod + def _send(conn, status, headers, body=b""): + lines = [f"HTTP/1.1 {status} X"] + [f"{k}: {v}" for k, v in headers] + conn.sendall(("\r\n".join(lines) + "\r\n\r\n").encode() + body) + + def _error(self, conn, status, size=None): + h = [("Content-Length", "0")] + if status == 405: + h.append(("Allow", "HEAD, POST")) + if status == 416: + h.append(("Content-Range", f"bytes */{size}")) + self._send(conn, status, h) + + def _handle(self, conn) -> None: + fds: list[int] = [] + try: + head, rest, fds = self._recv_head(conn) + if head is None: + return + lines = head.split("\r\n") + method, target, _ = lines[0].split(" ") + headers = {} + for line in lines[1:]: + k, _, v = line.partition(":") + headers[k.strip().lower()] = v.strip() + kind = None + if len(fds) == 1: + kind = "regular" if stat.S_ISREG(os.fstat(fds[0]).st_mode) else "other" + seen = Seen(method, target, headers, len(fds), kind) + self.seen.append(seen) + self._dispatch(conn, method, target, headers, fds, seen, rest) + except OSError: + pass # the client went away, as tests for bad frames expect + finally: + for fd in fds: + os.close(fd) + conn.close() + + def _dispatch(self, conn, method, target, headers, fds, seen, rest): + if headers.get("host") != "racer" or not target.startswith(wire.OBJECT_PREFIX): + self._error(conn, 400) + return + tail = target[len(wire.OBJECT_PREFIX) :] + key, _, mode = tail.partition("/") + if mode and (self.old or mode not in ("host-read", "gpu-read")): + self._error(conn, 400) + return + # Order follows the dataplane: RequestParser::parse + # (object-wire/src/client.rs), then direct::open_sink and + # HostSink::open (src/client/direct.rs), then the object read. + if mode and method != "POST": + self._error(conn, 405) + return + if mode: + if headers.get("content-length") != "0": + self._error(conn, 400) + return + try: + off = wire.decimal(headers.get("racer-destination-offset")) + length = wire.decimal(headers.get("racer-destination-length")) + alloc = wire.decimal(headers.get("racer-allocation-size")) if mode == "gpu-read" else None + except racer.RacerError: + self._error(conn, 400) + return + if length == 0 or off + length > wire.MAX_WIRE_INTEGER or (alloc is not None and off + length > alloc): + self._error(conn, 400) + return + if fds and not mode: + self._error(conn, 400) + return + if mode and len(fds) != 1: + self._error(conn, 400) + return + if mode == "gpu-read" and self.gpu_status != 200: + self._error(conn, self.gpu_status) + return + if mode == "host-read" and self.host_status: + self._error(conn, self.host_status) + return + if mode == "host-read": + flags = fcntl.fcntl(fds[0], fcntl.F_GETFL) + if seen.fd_kind != "regular" or flags & os.O_ACCMODE == os.O_RDONLY or flags & os.O_APPEND: + self._error(conn, 400) + return + obj = self.objects.get(key) + if obj is None: + self._error(conn, 404) + return + if "if-match" in headers and headers["if-match"] != obj.etag: + self._error(conn, 412) + return + meta = [("ETag", obj.etag), ("Racer-Expires-At", str(obj.expires_ms))] + if obj.content_type: + meta.append(("Racer-Content-Type", obj.content_type)) + size = len(obj.data) + if method == "HEAD": + self._send( + conn, 200, meta + [("Content-Length", str(size)), ("Content-Type", "application/octet-stream")] + ) + return + if method != "POST": + self._error(conn, 405) + return + first, end = 0, size + if "range" in headers: + spec = headers["range"][len("bytes=") :] + a, _, b = spec.partition("-") + first = int(a) + end = min(size, int(b) + 1) if b else size + if first >= size and size > 0 or first > size: + self._error(conn, 416, size) + return + pages = wire.page_count(first, end) + sel = meta + [ + ("Content-Type", "application/octet-stream"), + ("Racer-Object-Length", str(size)), + ("Racer-Range-Start", str(first)), + ("Racer-Range-End", str(end)), + ("Connection", "close"), + ] + if mode: + if length < end - first: + self._error(conn, 400) + return + self._send( + conn, + 200, + sel + + [ + ("Racer-Delivery", "host" if mode == "host-read" else "gpu"), + ("Content-Length", str(wire.FRAME_SIZE * (pages + 1))), + ], + ) + for f in self._page_frames(first, end): + chunk = obj.data[f.offset : f.offset + f.length] + if mode == "host-read": + os.pwrite(fds[0], chunk, off + f.offset - first) + else: + self.gpu_write(fds[0], off + f.offset - first, chunk) + conn.sendall(wire.encode_frame(f.kind, f.number, f.offset, f.length)) + if self.truncate: + return + conn.sendall(wire.encode_frame(wire.COMPLETE_FRAME, pages, end - first, 0)) + return + body = end - first + wire.FRAME_SIZE * (pages + 1) + self._send(conn, 200, sel + [("Content-Length", str(body))]) + credits = rest or b"" + inflight = 0 + for i, f in enumerate(self._page_frames(first, end)): + number = f.number + (1 if self.bad_frame and i == 1 else 0) + conn.sendall(wire.encode_frame(f.kind, number, f.offset, f.length)) + conn.sendall(obj.data[f.offset : f.offset + f.length]) + inflight += 1 + if inflight >= 2: + credits = self._credit(conn, credits, seen) + inflight -= 1 + while inflight: + credits = self._credit(conn, credits, seen) + inflight -= 1 + if self.truncate: + return + conn.sendall(wire.encode_frame(wire.COMPLETE_FRAME, pages, end - first, 0)) + + def _credit(self, conn, buf, seen): + while len(buf) < wire.CREDIT_SIZE: + got = conn.recv(64) + if not got: + raise ConnectionError("client closed") + buf += got + seen.credits.append(wire.decode_credit(buf[: wire.CREDIT_SIZE])) + return buf[wire.CREDIT_SIZE :] + + @staticmethod + def _page_frames(first, end): + if end <= first: + return + n = first // wire.PAGE_SIZE + o = first + while o < end: + e = min(end, (n + 1) * wire.PAGE_SIZE) + yield wire.Frame(wire.PAGE_FRAME, n, o, e - o) + n += 1 + o = e diff --git a/clients/python/racer/tests/test_client.py b/clients/python/racer/tests/test_client.py new file mode 100644 index 0000000000..071daee8f6 --- /dev/null +++ b/clients/python/racer/tests/test_client.py @@ -0,0 +1,322 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +import os + +import pytest +from conftest import FakeGpu +from fakeserver import Obj + +import racer +import racer._wire as wire +import racer.client + +DATA = bytes(range(256)) * 50 # 12800 bytes: 4 pages of 4096 (last partial) + + +def add(server, data=DATA, etag='"v1"', name="obj"): + req = racer.Request(racer.object_request("bucket", name).key) + server.objects[req.hex] = Obj(data, etag, "text/plain") + return req + + +def client(server, **kw): + kw.setdefault("staging_bytes", 3 * 4096) + return racer.Client(socket_path=server.path, **kw) + + +def test_socket_path(monkeypatch): + monkeypatch.delenv("RACER_SOCKET", raising=False) + assert racer.socket_path_for("c1") == "/run/racer/c1/client/socket" + assert racer.socket_path_for("c1", "/x") == "/x" + monkeypatch.setenv("RACER_SOCKET", "/env") + assert racer.socket_path_for("c1") == "/env" + monkeypatch.delenv("RACER_SOCKET") + with pytest.raises(racer.RacerError): + racer.socket_path_for(None) + with pytest.raises(racer.RacerError): + racer.socket_path_for("a/b") + + +def test_stat(server): + req = add(server) + m = client(server).stat(req) + assert (m.etag, m.size, m.content_type, m.expires_at_ms) == ('"v1"', len(DATA), "text/plain", 1_700_000_000_000) + seen = server.seen[-1] + assert seen.method == "HEAD" and seen.headers["host"] == "racer" + assert seen.headers["racer-metadata"] == req.metadata if req.metadata else True + with pytest.raises(racer.VersionUnavailable): + client(server).stat(req, if_match='"v2"') + with pytest.raises(racer.NotFound): + client(server).stat(racer.Request(b"\0" * 32)) + + +def test_stream_read_full_and_range(server): + req = add(server) + c = client(server) + assert c.read(req) == DATA + seen = server.seen[-1] + assert seen.headers["racer-page-credits"] == "2" + assert seen.headers["racer-byte-credits"] == str(2 * 16 * 1024 * 1024) + assert "range" not in seen.headers + # Every page's credit comes back, including the last one. + assert seen.credits == [(0, 4096), (1, 4096), (2, 4096), (3, 512)] + assert c.read(req, offset=100, length=5000) == DATA[100:5100] + assert server.seen[-1].headers["range"] == "bytes=100-5099" + assert c.read(req, offset=5000) == DATA[5000:] + assert server.seen[-1].headers["range"] == "bytes=5000-" + + +def test_stream_empty_object(server): + req = add(server, data=b"") + assert client(server).read(req) == b"" + + +def test_stream_short_object_is_unsatisfiable(server): + req = add(server) + with pytest.raises(racer.UnsatisfiableRange): + client(server).read(req, offset=len(DATA) - 10, length=100) + with pytest.raises(racer.UnsatisfiableRange): + client(server).read(req, offset=len(DATA) + 1) + + +def test_stream_bad_frame(server): + req = add(server) + server.bad_frame = True + with pytest.raises(racer.ProtocolError): + client(server).read(req) + + +def test_stream_truncated(server): + req = add(server) + server.truncate = True + with pytest.raises(racer.TruncatedError): + client(server).read(req) + + +def test_if_match_sent(server): + req = add(server) + assert client(server).read(req, if_match='"v1"') == DATA + assert server.seen[-1].headers["if-match"] == '"v1"' + with pytest.raises(racer.VersionUnavailable): + client(server).read(req, if_match='"v0"') + with pytest.raises(racer.RacerError): + client(server).read(req, if_match="unquoted") + + +def test_host_read_passes_memfd(server): + req = add(server) + pages = [] + with racer.HostBuffer(len(DATA) + 10) as buf: + r = client(server).read_into_host(req, buf, 10, on_page=lambda o, n: pages.append((o, n))) + assert r.delivery == racer.HOST and r.pages == 4 + assert bytes(buf.view[10:]) == DATA + seen = server.seen[-1] + assert seen.target.endswith("/host-read") + assert seen.fds == 1 and seen.fd_kind == "regular" + assert seen.headers["racer-destination-offset"] == "10" + assert seen.headers["racer-destination-length"] == str(len(DATA)) + assert "racer-allocation-size" not in seen.headers + assert pages == [(0, 4096), (4096, 4096), (8192, 4096), (12288, 512)] + + +def test_host_read_range(server): + req = add(server) + with racer.HostBuffer(5000) as buf: + r = client(server).read_into_host(req, buf, offset=4000, length=5000) + assert (r.first, r.end) == (4000, 9000) + assert bytes(buf.view) == DATA[4000:9000] + + +def test_old_dataplane_falls_back_to_stream_and_remembers(server): + req = add(server) + server.old = True + c = client(server) + with racer.HostBuffer(len(DATA)) as buf: + r = c.read_into_host(req, buf) + assert r.delivery == racer.STREAM + assert bytes(buf.view) == DATA + assert c.mode_state(racer.HOST) == "unsupported" + n = len(server.seen) + r = c.read_into_host(req, buf) + assert r.delivery == racer.STREAM + # The second read goes straight to stream mode. + assert len(server.seen) == n + 1 and "/host-read" not in server.seen[-1].target + + +def test_missing_object_does_not_disable_mode(server): + c = client(server) + with racer.HostBuffer(10) as buf: + with pytest.raises(racer.NotFound): + c.read_into_host(racer.Request(b"\1" * 32), buf) + assert c.mode_state(racer.HOST) is None + + +def test_supported_mode_errors_are_not_fallbacks(server): + req = add(server) + c = client(server) + with racer.HostBuffer(len(DATA)) as buf: + c.read_into_host(req, buf) + assert c.mode_state(racer.HOST) == "supported" + server.host_status = 400 + with pytest.raises(racer.StatusError): + c.read_into_host(req, buf) + + +def test_plain_buffer_uses_stream(server): + req = add(server) + out = bytearray(len(DATA)) + r = client(server).read_into_host(req, out) + assert r.delivery == racer.STREAM and bytes(out) == DATA + assert server.seen[-1].fds == 0 + + +def test_destination_too_small(server): + req = add(server) + with racer.HostBuffer(100) as buf: + # host-read answers 400, which looks like an old dataplane, so the + # stream fallback reports the size mismatch. + c = client(server) + with pytest.raises(racer.RacerError, match="smaller"): + c.read_into_host(req, buf) + # The refusal was about this read, not about host-read support. + assert c.mode_state(racer.HOST) is None + out = bytearray(100) + with pytest.raises(racer.RacerError): + client(server).read_into_host(req, out) + + +def test_gpu_read_success(server, gpu_writer): + req = add(server) + server.gpu_status = 200 + server.gpu_write = gpu_writer + g = FakeGpu(len(DATA) + 100) + try: + r = client(server).read_into_gpu(req, g, 100) + assert r.delivery == racer.GPU + assert g.read(100, len(DATA)) == DATA + seen = server.seen[-1] + assert seen.target.endswith("/gpu-read") and seen.fds == 1 + assert seen.headers["racer-allocation-size"] == str(g.size) + assert seen.headers["racer-destination-length"] == str(len(DATA)) + assert g.copies == 0 + finally: + g.close() + + +def test_gpu_falls_back_to_host_chunks(server): + req = add(server) + g = FakeGpu(len(DATA)) + pages = [] + try: + c = client(server) + r = c.read_into_gpu(req, g, on_page=lambda o, n: pages.append((o, n))) + assert r.delivery == racer.HOST and (r.first, r.end, r.pages) == (0, len(DATA), 4) + assert g.read(0, len(DATA)) == DATA + assert c.mode_state(racer.GPU) == "unsupported" + assert pages == [(0, 4096), (4096, 4096), (8192, 4096), (12288, 512)] + reads = [s for s in server.seen if s.target.endswith("/host-read")] + # Staging holds 3 pages, so the object takes two host-reads. + assert len(reads) == 2 and reads[1].headers["if-match"] == '"v1"' + finally: + g.close() + + +def test_gpu_falls_back_to_stream_on_old_dataplane(server): + req = add(server) + server.old = True + g = FakeGpu(len(DATA)) + try: + c = client(server) + r = c.read_into_gpu(req, g, offset=10, length=9000) + assert r.delivery == racer.STREAM + assert g.read(0, 9000) == DATA[10:9010] + assert c.mode_state(racer.GPU) == "unsupported" + assert c.mode_state(racer.HOST) == "unsupported" + finally: + g.close() + + +def test_gpu_not_exportable_skips_without_memory(server): + req = add(server) + g = FakeGpu(len(DATA), exportable=False) + try: + c = client(server) + r = c.read_into_gpu(req, g) + assert r.delivery == racer.HOST + assert c.mode_state(racer.GPU) is None + finally: + g.close() + + +def test_modes_remembered_per_socket(server, tmp_path): + req = add(server) + server.old = True + with racer.HostBuffer(len(DATA)) as buf: + client(server).read_into_host(req, buf) + assert racer.client._mode_state[server.path][racer.HOST] == "unsupported" + other = racer.Client(socket_path=str(tmp_path / "other")) + assert other.mode_state(racer.HOST) is None + + +def test_send_fd_with_head(server): + """The descriptor arrives with the head even when nothing else is sent.""" + req = add(server) + with racer.HostBuffer(len(DATA)) as buf: + c = client(server) + c.host_read(req, buf.fileno(), 0, len(DATA)) + assert server.seen[-1].fds == 1 + assert bytes(buf.view) == DATA + # Our copy of the descriptor is still open and owned by the caller. + fd = os.memfd_create("x") + c.host_read(req, fd, 0, len(DATA)) + os.fstat(fd) + os.close(fd) + + +def test_gpu_read_window_check(server): + req = add(server) + with pytest.raises(racer.RacerError): + client(server).gpu_read(req, 0, 100, 50, 60) + + +def test_frame_sequence_rules(small_pages): + s = wire.Sequence(100, 9000) + with pytest.raises(racer.ProtocolError): + s.accept(wire.Frame(1, 0, 0, 4096)) # starts before range + s = wire.Sequence(100, 9000) + s.accept(wire.Frame(1, 0, 100, 3996)) + with pytest.raises(racer.ProtocolError): + s.accept(wire.Frame(1, 2, 8192, 808)) # skipped a page + s = wire.Sequence(100, 9000) + s.accept(wire.Frame(1, 0, 100, 3996)) + s.accept(wire.Frame(1, 1, 4096, 4096)) + with pytest.raises(racer.ProtocolError): + s.accept(wire.Frame(2, 3, 8900, 0)) # completion too early + s.accept(wire.Frame(1, 2, 8192, 808)) + with pytest.raises(racer.ProtocolError): + s.accept(wire.Frame(2, 3, 8900, 1)) # nonzero length + s.accept(wire.Frame(2, 3, 8900, 0)) + with pytest.raises(racer.ProtocolError): + s.accept(wire.Frame(2, 3, 8900, 0)) # after completion + assert wire.decode_frame(wire.encode_frame(1, 2, 3, 4)) == wire.Frame(1, 2, 3, 4) + assert wire.encode_credit(1, 2) == b"\0" * 7 + b"\1" + b"\0\0\0\2" + + +def test_decimal(): + assert wire.decimal("0") == 0 + for bad in ["", "01", "-1", "1a", "9" * 20, str(1 << 63)]: + with pytest.raises(racer.ProtocolError): + wire.decimal(bad) + + +def test_response_head_parsing(): + h = wire.parse_response_head(b"HTTP/1.1 200 OK\r\nETag: \"a\"\r\nConnection: keep-alive, close\r\n\r\n") + assert h.status == 200 and h.get("ETag") == '"a"' and wire.connection_close(h) + with pytest.raises(racer.ProtocolError): + wire.parse_response_head(b"HTTP/1.1 200 OK\r\nETag: a\r\nETag: b\r\n\r\n") + with pytest.raises(racer.ProtocolError): + wire.parse_response_head(b"HTTP/1.0 200 OK\r\n\r\n") + with pytest.raises(racer.ProtocolError): + wire.check_error_status(wire.parse_response_head(b"HTTP/1.1 404 N\r\nContent-Length: 3\r\n\r\n")) + with pytest.raises(racer.ProtocolError): + wire.check_error_status(wire.parse_response_head(b"HTTP/1.1 416 N\r\nContent-Length: 0\r\n\r\n")) diff --git a/clients/python/racer/tests/test_cuda.py b/clients/python/racer/tests/test_cuda.py new file mode 100644 index 0000000000..406803abe5 --- /dev/null +++ b/clients/python/racer/tests/test_cuda.py @@ -0,0 +1,260 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +import ctypes + +import pytest + +from racer import cuda + + +def test_struct_layouts(): + # Sizes and offsets of the cuda.h structs on LP64 Linux. + assert ctypes.sizeof(cuda.CUmemLocation) == 8 + assert ctypes.sizeof(cuda.CUmemAllocationProp) == 32 + assert cuda.CUmemAllocationProp.location.offset == 8 + assert cuda.CUmemAllocationProp.win32HandleMetaData.offset == 16 + assert cuda.CUmemAllocationProp.allocFlags.offset == 24 + assert ctypes.sizeof(cuda.CUmemAccessDesc) == 12 + + +def test_missing_libcuda(monkeypatch): + monkeypatch.setenv("RACER_LIBCUDA", "libdoes-not-exist.so") + monkeypatch.setattr(cuda, "_lib", None) + assert not cuda.available() + with pytest.raises(cuda.CudaUnavailable): + cuda.GpuAllocation(1) + + +def test_host_buffer_pin_without_cuda(monkeypatch): + from racer import HostBuffer + + monkeypatch.setenv("RACER_LIBCUDA", "libdoes-not-exist.so") + monkeypatch.setattr(cuda, "_lib", None) + with HostBuffer(4096) as b: + assert b.pin() is False and not b.pinned + b.view[:3] = b"abc" + assert bytes(b.view[:3]) == b"abc" + + +@pytest.mark.skipif(not cuda.available(), reason="needs a GPU") +def test_gpu_allocation_roundtrip(): + with cuda.GpuAllocation(1000) as g: + assert g.size >= 1000 + g.copy_from_host(0, b"hello") + out = bytearray(5) + g.copy_to_host(0, out) + assert out == b"hello" + import os + + fd = g.export_fd() + assert fd >= 0 + os.close(fd) + + +class _FakeLib: + """Records driver calls; contexts are ints, primary context of device d is 100 + d.""" + + def __init__(self, current: int = 0, fail: str | None = None) -> None: + self.current = current + self.fail = fail + self.calls: list[tuple] = [] + + def _rc(self, name: str) -> int: + return 1 if name == self.fail else 0 + + def cuDeviceGet(self, out, ordinal): + out._obj.value = ordinal + return 0 + + def cuDevicePrimaryCtxRetain(self, out, dev): + out._obj.value = 100 + dev + self.calls.append(("retain", dev)) + return 0 + + def cuCtxGetCurrent(self, out): + out._obj.value = self.current or None + return 0 + + def cuCtxSetCurrent(self, ctx): + value = ctx.value if isinstance(ctx, ctypes.c_void_p) else ctx + self.current = value or 0 + self.calls.append(("set", self.current)) + return 0 + + def cuMemcpyHtoD_v2(self, dst, src, n): + self.calls.append(("htod", self.current)) + return self._rc("cuMemcpyHtoD_v2") + + def cuMemcpyDtoH_v2(self, dst, src, n): + self.calls.append(("dtoh", self.current)) + return self._rc("cuMemcpyDtoH_v2") + + def cuMemHostRegister_v2(self, addr, n, flags): + self.calls.append(("register", self.current)) + return self._rc("cuMemHostRegister_v2") + + def cuMemHostUnregister(self, addr): + return 0 + + +@pytest.fixture +def fake(monkeypatch): + def make(**kw): + lib = _FakeLib(**kw) + monkeypatch.setattr(cuda, "_lib", lib) + monkeypatch.setattr(cuda, "_contexts", {}) + return lib + + return make + + +def _alloc(device: int) -> cuda.GpuAllocation: + # Skip __init__ (VMM calls); only the fields the copy paths use. + g = cuda.GpuAllocation.__new__(cuda.GpuAllocation) + g.device, g.size, g.ptr = device, 4096, 0x1000 + g._handle, g._reserved, g._mapped = None, False, False + return g + + +@pytest.mark.parametrize("current", [0, 100, 101]) +def test_copy_from_host_binds_allocation_context(fake, current): + lib = fake(current=current) + _alloc(2).copy_from_host(0, b"abc") + assert ("htod", 102) in lib.calls + assert lib.current == current + if current: + assert lib.calls[-1] == ("set", current) + + +def test_copy_from_host_restores_context_on_error(fake): + lib = fake(current=100, fail="cuMemcpyHtoD_v2") + with pytest.raises(cuda.CudaError): + _alloc(1).copy_from_host(0, b"abc") + assert lib.calls[-2:] == [("htod", 101), ("set", 100)] + assert lib.current == 100 + + +def test_copy_from_host_without_prior_context(fake): + lib = fake(current=0) + _alloc(1).copy_from_host(0, b"abc") + assert lib.calls == [("retain", 1), ("set", 101), ("htod", 101), ("set", 0)] + + +def test_copy_to_host_binds_allocation_context(fake): + lib = fake(current=100) + _alloc(3).copy_to_host(0, bytearray(3)) + assert ("dtoh", 103) in lib.calls and lib.current == 100 + + +def test_already_current_context_is_not_switched(fake): + lib = fake(current=101) + _alloc(1).copy_from_host(0, b"abc") + assert [c for c in lib.calls if c[0] == "set"] == [] + + +def test_host_register_uses_requested_device(fake): + lib = fake(current=0) + cuda.host_register(bytearray(64), 2).close() + assert ("register", 102) in lib.calls and lib.current == 0 + + +def test_host_register_restores_context_on_error(fake): + lib = fake(current=100, fail="cuMemHostRegister_v2") + with pytest.raises(cuda.CudaError): + cuda.host_register(bytearray(64), 1) + assert lib.current == 100 + + +def test_failed_pin_releases_exports_and_closes_memfd(fake): + import os + + from racer import HostBuffer + + fake(fail="cuMemHostRegister_v2") + with pytest.raises(cuda.CudaError) as ei: + with HostBuffer(4096) as b: + fd = b._fd + b.pin(0) + assert "cuMemHostRegister_v2" in str(ei.value) + assert b._map.closed + with pytest.raises(OSError): + os.fstat(fd) + + +def test_host_register_releases_export_on_failure(fake): + import mmap + + fake(fail="cuMemHostRegister_v2") + m = mmap.mmap(-1, 4096) + with pytest.raises(cuda.CudaError): + cuda.host_register(m, 0) + m.close() + + +def test_staging_pinned_for_destination_device(fake, tmp_path): + from racer.client import Client + + lib = fake(current=0) + c = Client.__new__(Client) + c._pool, c.staging_bytes, c.pin_staging = [], 4096, True + import threading + + c._pool_lock = threading.Lock() + buf = c._take_staging(3) + try: + assert ("register", 103) in lib.calls + finally: + buf.close() + + +def _stub_torch(monkeypatch): + import sys + import types + + torch = types.ModuleType("torch") + torch.uint8 = "uint8" + monkeypatch.setitem(sys.modules, "torch", torch) + + +class _StubTensor: + def __init__(self, index): + self.dtype = "uint8" + self.device = type("Dev", (), {"type": "cuda", "index": index})() + + def dim(self): + return 1 + + def is_contiguous(self): + return True + + def numel(self): + return 16 + + +def test_tensor_target_exposes_device(monkeypatch): + from racer.client import _dest_device + from racer.tensors import TensorTarget + + _stub_torch(monkeypatch) + assert TensorTarget(_StubTensor(3)).device == 3 + assert _dest_device(TensorTarget(_StubTensor(3))) == 3 + assert _dest_device(TensorTarget(_StubTensor(None))) is None + + +def test_tensor_target_staging_on_tensor_device(fake, monkeypatch): + import threading + + from racer.client import Client, _dest_device + from racer.tensors import TensorTarget + + _stub_torch(monkeypatch) + lib = fake(current=0) + c = Client.__new__(Client) + c._pool, c.staging_bytes, c.pin_staging = [], 4096, True + c._pool_lock = threading.Lock() + buf = c._take_staging(_dest_device(TensorTarget(_StubTensor(3)))) + try: + assert ("register", 103) in lib.calls + assert ("register", 100) not in lib.calls + finally: + buf.close() diff --git a/clients/python/racer/tests/test_keys.py b/clients/python/racer/tests/test_keys.py new file mode 100644 index 0000000000..912bf8957e --- /dev/null +++ b/clients/python/racer/tests/test_keys.py @@ -0,0 +1,63 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +import pytest + +import racer + +# Computed with internal/racerobject.NewRequest (Go encoding/json + SHA-256). +VECTORS = [ + ( + "models", + "llama/config.json", + "", + "3af125b4fe06cdb5fc810214a357c138d4acfc54d99ef56791f805182d4044e0", + '{"schema":1,"bucket":"models","key":"llama/config.json"}', + ), + ( + "models", + "llama/model-00001-of-00002.safetensors", + "v1", + "982a88e7fe93c3d53ff66b9045aa99089251f66d613790c6c5888b97054bdd16", + '{"schema":1,"bucket":"models","key":"llama/model-00001-of-00002.safetensors","versionId":"v1"}', + ), + ( + "b.u-c_k", + 'a&c "q" \\ \n\t\b\f\x01\x7f \u00e9 \u65e5\u672c \u2028\u2029 \U0001f600', + "3HL4kqtJlcpXroDTDmJ+rmSpXd3dIbrHY+MTRCxf3vjVBH40Nr8X8gdRQBpUMLUo", + "fde698046567709f2f289be4e4dc246b9fbc40a04b4df1ca0f3cf2d6e891b1ac", + None, + ), +] + + +@pytest.mark.parametrize("bucket,key,version,want,canonical", VECTORS) +def test_object_key_vectors(bucket, key, version, want, canonical): + assert racer.object_key(bucket, key, version) == want + if canonical is not None: + assert racer.canonical_json(bucket, key, version) == canonical + r = racer.object_request(bucket, key, version or None) + assert r.hex == want and r.metadata == racer.canonical_json(bucket, key, version) + + +@pytest.mark.parametrize( + "bucket,key,version", + [ + ("", "k", ""), + (".", "k", ""), + ("a/b", "k", ""), + ("b", "", ""), + ("b", "x" * 1025, ""), + ("b", "k", "v" * 1025), + ("b", "\ud800", ""), + ], +) +def test_object_key_invalid(bucket, key, version): + with pytest.raises(racer.RacerError): + racer.object_key(bucket, key, version) + + +def test_request_from_hex(): + r = racer.Request.from_hex("ab" * 32) + assert r.key == b"\xab" * 32 + with pytest.raises(racer.RacerError): + racer.Request.from_hex("AB" * 32) diff --git a/clients/python/racer/tests/test_wire_golden.py b/clients/python/racer/tests/test_wire_golden.py new file mode 100644 index 0000000000..a8d5a22466 --- /dev/null +++ b/clients/python/racer/tests/test_wire_golden.py @@ -0,0 +1,119 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Golden bytes shared with the dataplane. + +Sources: cmd/racer-dataplane/object-wire/src/client.rs (RequestParser::parse, +direct_head, frame) and object-wire/tests/direct.rs. +""" + +import array +import fcntl +import os +import socket + +import pytest + +import racer +import racer._wire as wire + +KEY = "ab" * 32 + + +def test_frame_layout_matches_rust_frame(): + # frame(kind, number, offset, length): u8, u64 BE, u64 BE, u32 BE. + raw = wire.encode_frame(1, 0x0102030405060708, 0x1112131415161718, 0x21222324) + assert raw == bytes([1, 1, 2, 3, 4, 5, 6, 7, 8, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x21, 0x22, 0x23, 0x24]) + # Direct completion is frame(2, pages, sent, 0). + assert wire.decode_frame(wire.encode_frame(2, 3, 2 * wire.PAGE_SIZE + 5, 0)) == wire.Frame(2, 3, 2 * wire.PAGE_SIZE + 5, 0) + + +def _capture(mode, call): + """Run one RPC against a socketpair-like listener and return head bytes and fds.""" + import tempfile + import threading + + d = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) + path = os.path.join(d, "s") + lst = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + lst.bind(path) + lst.listen(1) + got = {} + + def serve(): + conn, _ = lst.accept() + # The first recvmsg must carry the descriptor with the first head bytes. + data, anc, _, _ = conn.recvmsg(65536, socket.CMSG_SPACE(4 * 8)) + fds = array.array("i") + for level, typ, cdata in anc: + if level == socket.SOL_SOCKET and typ == socket.SCM_RIGHTS: + fds.frombytes(cdata[: len(cdata) - len(cdata) % 4]) + while b"\r\n\r\n" not in data: + data += conn.recv(65536) + got["head"] = data + got["flags"] = [fcntl.fcntl(fd, fcntl.F_GETFL) for fd in fds] + for fd in fds: + os.close(fd) + conn.sendall(b"HTTP/1.1 501 X\r\nContent-Length: 0\r\n\r\n") + conn.close() + + t = threading.Thread(target=serve) + t.start() + c = racer.Client(socket_path=path) + with pytest.raises(racer.StatusError): + call(c) + t.join(5) + lst.close() + os.unlink(path) + os.rmdir(d) + return got + + +def test_host_read_request_bytes(): + req = racer.Request(bytes.fromhex(KEY)) + with racer.HostBuffer(4196) as buf: + got = _capture( + "host", + lambda c: c.host_read(req, buf.fileno(), 4096, 100, offset=1, length=3, if_match='"v1"'), + ) + assert got["head"] == ( + f"POST /v1/objects/{KEY}/host-read HTTP/1.1\r\nHost: racer\r\nContent-Length: 0\r\n" + 'Range: bytes=1-3\r\nIf-Match: "v1"\r\n' + "Racer-Destination-Offset: 4096\r\nRacer-Destination-Length: 100\r\n\r\n" + ).encode() + # HostSink::open needs a writable, non-append descriptor. + (flags,) = got["flags"] + assert flags & os.O_ACCMODE == os.O_RDWR and not flags & os.O_APPEND + + +def test_gpu_read_request_bytes(): + req = racer.Request(bytes.fromhex(KEY)) + with racer.HostBuffer(1) as buf: + got = _capture("gpu", lambda c: c.gpu_read(req, buf.fileno(), 4196, 4096, 100)) + assert got["head"] == ( + f"POST /v1/objects/{KEY}/gpu-read HTTP/1.1\r\nHost: racer\r\nContent-Length: 0\r\n" + "Racer-Destination-Offset: 4096\r\nRacer-Destination-Length: 100\r\nRacer-Allocation-Size: 4196\r\n\r\n" + ).encode() + assert len(got["flags"]) == 1 + + +def test_direct_head_from_rust_is_accepted(): + # direct_head for a 3-byte range of an object of 2 pages + 5 bytes: + # the Get head with Content-Length 21 * (pages + 1) and Racer-Delivery + # inserted before Connection. + length = 2 * wire.PAGE_SIZE + 5 + raw = ( + 'HTTP/1.1 200 OK\r\nETag: "v1"\r\nRacer-Expires-At: 1700000000000\r\n' + "Content-Type: application/octet-stream\r\nRacer-Object-Length: %d\r\n" + "Racer-Range-Start: 3\r\nRacer-Range-End: 6\r\nContent-Length: 42\r\n" + "Racer-Delivery: host\r\nConnection: close\r\n\r\n" % length + ).encode() + h = wire.parse_response_head(raw) + import racer.client + + meta, first, end = racer.client.Client._check_selection(h, 3, 3, None) + assert (meta.size, first, end) == (length, 3, 6) + assert wire.decimal(h.get("content-length")) == wire.FRAME_SIZE * (wire.page_count(first, end) + 1) + seq = wire.Sequence(first, end) + seq.accept(wire.decode_frame(wire.encode_frame(1, 0, 3, 3))) + seq.accept(wire.decode_frame(wire.encode_frame(2, 1, 3, 0))) + assert seq.complete diff --git a/cmd/racer-dataplane/.client-test-2179449-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-11/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-12/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-12/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-13/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-13/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-15/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-15/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-17/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-18/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-20/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-21/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-22/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-23/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-24/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-24/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-25/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-33/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-34/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-37/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-37/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-38/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-39/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-39/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-4/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-40/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-40/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-42/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-46/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-46/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-5/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-57/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-57/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-58/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-58/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-59/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-59/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-60/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-60/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-61/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-61/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-65/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-65/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-69/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-69/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-8/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179449-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-9/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-10/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-10/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-11/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-12/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-12/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-13/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-13/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-14/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-16/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-18/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-2/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-2/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-20/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-21/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-22/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-23/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-24/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-24/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-25/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-26/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-26/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-29/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-29/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-30/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-30/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-33/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-35/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-36/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-37/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-37/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-38/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-39/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-39/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-4/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-41/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-42/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-43/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-43/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-5/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-52/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-52/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-6/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-6/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-60/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-60/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-62/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-62/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-7/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-7/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-8/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2179709-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-9/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-10/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-10/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-11/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-14/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-17/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-18/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-20/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-21/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-22/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-25/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-26/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-26/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-28/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-29/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-29/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-33/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-36/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-38/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-39/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-39/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-4/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-42/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-47/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-47/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-57/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-57/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-58/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-58/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-61/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-61/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-62/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-62/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-63/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-63/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-67/added/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-67/added/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-67/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-67/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-68/blocked b/cmd/racer-dataplane/.client-test-2180946-68/blocked new file mode 100644 index 0000000000..2a5e269186 --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2180946-68/blocked @@ -0,0 +1 @@ +foreign \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2180946-68/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-68/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-69/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-69/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-69/example/origin/socket b/cmd/racer-dataplane/.client-test-2180946-69/example/origin/socket new file mode 100644 index 0000000000..6fa78783bf --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2180946-69/example/origin/socket @@ -0,0 +1 @@ +origin \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2180946-70/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-70/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-71/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-71/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-75/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-75/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-8/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2180946-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-9/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-10/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-10/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-11/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-16/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-17/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-18/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-2/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-2/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-20/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-21/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-22/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-23/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-25/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-26/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-26/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-28/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-29/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-29/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-31/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-31/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-33/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-34/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-39/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-39/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-4/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-41/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-42/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-43/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-43/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-45/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-45/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-5/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-57/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-57/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-58/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-58/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-59/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-59/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-6/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-6/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-60/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-60/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-7/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-7/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-8/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181225-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-9/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-10/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-10/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-11/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-16/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-17/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-18/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-2/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-2/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-21/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-22/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-23/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-24/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-24/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-25/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-27/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-27/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-28/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-30/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-30/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-33/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-34/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-35/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-36/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-37/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-37/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-38/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-39/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-39/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-4/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-40/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-40/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-41/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-44/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-44/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-55/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-55/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-56/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-56/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-59/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-59/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-6/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-6/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-63/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-63/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-63/other/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-63/other/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-65/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-65/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-67/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-67/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-8/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2181449-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-9/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-10/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-10/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-13/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-13/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-14/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-16/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-17/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-18/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-2/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-2/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-21/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-22/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-23/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-24/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-24/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-25/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-26/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-26/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-28/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-29/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-29/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-33/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-34/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-35/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-36/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-38/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-39/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-39/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-4/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-40/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-40/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-41/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-43/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-43/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-47/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-47/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-5/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-8/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2182990-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-9/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-10/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-10/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-11/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-12/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-12/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-14/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-16/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-17/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-18/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-2/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-2/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-20/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-23/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-24/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-24/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-25/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-26/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-26/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-28/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-33/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-34/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-35/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-38/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-39/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-39/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-4/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-41/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-42/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-5/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-6/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-6/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-60/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-60/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-62/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-62/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-7/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-7/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-8/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2183207-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-9/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-11/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-12/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-12/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-14/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-15/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-15/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-17/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-18/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-20/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-22/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-23/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-28/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-29/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-29/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-30/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-30/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-31/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-31/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-33/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-36/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-37/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-37/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-38/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-4/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-40/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-40/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-41/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-42/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-43/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-43/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-45/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-45/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-5/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-54/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-54/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-55/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-55/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-6/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-6/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-8/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184396-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-9/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-11/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-14/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-16/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-17/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-18/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-2/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-2/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-21/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-22/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-23/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-27/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-27/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-32/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-32/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-33/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-34/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-35/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-36/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-4/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-40/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-40/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-41/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-43/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-43/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-45/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-45/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-5/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-57/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-57/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-59/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-59/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-60/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-60/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-61/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-61/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-64/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-64/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2184635-68/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-68/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-102/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-102/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-105/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-105/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-14/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-15/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-15/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-16/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-20/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-22/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-24/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-24/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-36/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-37/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-37/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-38/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-41/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-46/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-46/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-47/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-47/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-48/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-48/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-5/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-51/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-51/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-54/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-54/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-58/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-58/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-61/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-61/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-63/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-63/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-65/added/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-65/added/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-65/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-65/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-67/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-67/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-67/example/origin/socket b/cmd/racer-dataplane/.client-test-2192245-67/example/origin/socket new file mode 100644 index 0000000000..6fa78783bf --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2192245-67/example/origin/socket @@ -0,0 +1 @@ +origin \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2192245-76/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-76/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-82/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-82/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-82/ready b/cmd/racer-dataplane/.client-test-2192245-82/ready new file mode 100644 index 0000000000..ad50b53407 --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2192245-82/ready @@ -0,0 +1 @@ +ready \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2192245-91/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-91/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-96/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-96/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2192245-98/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-98/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-16/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-17/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-18/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-21/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-25/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-39/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-39/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-4/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-42/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-43/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-43/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-57/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-57/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-60/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-60/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-62/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-62/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-65/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-65/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-66/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-66/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-67/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-67/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-69/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-69/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-71/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-71/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-72/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-72/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-72/other/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-72/other/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-73/added/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-73/added/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-73/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-73/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-74/blocked b/cmd/racer-dataplane/.client-test-2193402-74/blocked new file mode 100644 index 0000000000..2a5e269186 --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2193402-74/blocked @@ -0,0 +1 @@ +foreign \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2193402-74/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-74/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-78/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-78/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-79/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-79/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-82/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-82/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193402-84/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-84/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-14/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-16/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-17/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-18/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-21/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-22/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-23/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-27/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-27/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-28/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-30/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-30/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-38/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-41/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-42/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-43/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-43/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-44/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-44/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-45/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-45/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-49/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-49/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-54/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-54/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-57/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-57/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-6/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-6/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-60/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-60/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-63/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-63/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-65/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-65/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-67/blocked b/cmd/racer-dataplane/.client-test-2193712-67/blocked new file mode 100644 index 0000000000..2a5e269186 --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2193712-67/blocked @@ -0,0 +1 @@ +foreign \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2193712-67/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-67/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-68/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-68/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-69/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-69/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193712-70/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-70/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193981-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-14/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193981-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-17/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193981-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-18/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193981-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193981-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-21/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193981-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193981-37/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-37/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193981-44/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-44/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193981-54/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-54/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193981-55/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-55/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193981-73/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-73/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193981-80/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-80/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193981-81/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-81/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193981-81/ready b/cmd/racer-dataplane/.client-test-2193981-81/ready new file mode 100644 index 0000000000..ad50b53407 --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2193981-81/ready @@ -0,0 +1 @@ +ready \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2193981-82/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-82/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193981-83/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-83/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2193981-86/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-86/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-10/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-10/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-11/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-12/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-12/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-14/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-15/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-15/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-16/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-17/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-20/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-22/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-23/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-28/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-31/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-31/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-34/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-35/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-36/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-38/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-4/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-40/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-40/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-41/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-45/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-45/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-5/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-7/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-7/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194305-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-8/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-10/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-10/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-12/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-12/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-13/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-13/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-14/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-16/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-17/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-18/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-20/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-21/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-23/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-26/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-26/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-27/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-27/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-28/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-29/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-29/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-31/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-31/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-33/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-34/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-35/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-37/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-37/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-4/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-41/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-42/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-43/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-43/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-45/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-45/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-57/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-57/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-58/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-58/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-59/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-59/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-61/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-61/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-64/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-64/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-65/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-65/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-66/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-66/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-66/other/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-66/other/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-67/added/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-67/added/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-67/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-67/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-69/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-69/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-8/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194507-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-9/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-100/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-100/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-102/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-102/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-14/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-17/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-18/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-20/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-31/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-31/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-33/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-34/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-35/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-38/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-41/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-42/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-59/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-59/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-60/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-60/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-62/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-62/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-66/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-66/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-68/added/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-68/added/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-83/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-83/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-83/ready b/cmd/racer-dataplane/.client-test-2194756-83/ready new file mode 100644 index 0000000000..ad50b53407 --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2194756-83/ready @@ -0,0 +1 @@ +ready \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2194756-85/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-85/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-94/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-94/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2194756-97/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-97/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-14/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-17/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-18/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-21/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-25/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-26/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-26/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-27/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-27/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-28/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-29/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-29/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-30/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-30/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-33/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-35/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-36/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-38/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-4/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-42/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-45/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-45/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-5/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-59/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-59/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-61/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-61/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-63/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-63/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-64/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-64/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-66/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-66/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-67/added/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-67/added/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-67/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-67/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-68/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-68/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-68/other/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-68/other/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-70/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-70/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-71/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-71/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-71/example/origin/socket b/cmd/racer-dataplane/.client-test-2195107-71/example/origin/socket new file mode 100644 index 0000000000..6fa78783bf --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2195107-71/example/origin/socket @@ -0,0 +1 @@ +origin \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2195107-80/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-80/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195107-82/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-82/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-100/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-100/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-102/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-102/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-104/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-104/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-13/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-13/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-16/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-17/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-19/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-21/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-3/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-33/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-46/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-46/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-54/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-54/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-58/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-58/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-79/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-79/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-79/ready b/cmd/racer-dataplane/.client-test-2195404-79/ready new file mode 100644 index 0000000000..ad50b53407 --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2195404-79/ready @@ -0,0 +1 @@ +ready \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2195404-80/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-80/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-83/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-83/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-87/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-87/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-93/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-93/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-95/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-95/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/.client-test-2195404-97/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-97/example/client/.racer-client.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cmd/racer-dataplane/Cargo.lock b/cmd/racer-dataplane/Cargo.lock index f4f9f1beb5..4f73d3ee66 100644 --- a/cmd/racer-dataplane/Cargo.lock +++ b/cmd/racer-dataplane/Cargo.lock @@ -868,6 +868,7 @@ dependencies = [ "prost", "racer-control-wire", "racer-crypto", + "racer-gpu", "racer-object-wire", "rcgen", "rdma-verbs", @@ -885,6 +886,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "racer-gpu" +version = "0.1.0" +dependencies = [ + "cc", + "libc", + "racer-crypto", + "zeroize", +] + [[package]] name = "racer-object-wire" version = "0.1.0" diff --git a/cmd/racer-dataplane/Cargo.toml b/cmd/racer-dataplane/Cargo.toml index a21a68006b..1beab987b1 100644 --- a/cmd/racer-dataplane/Cargo.toml +++ b/cmd/racer-dataplane/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [".", "runtime", "alloc", "crypto", "verbs", "http", "topology", "telemetry", "flow", "control-wire", "controlplane", "uds-endpoint", "wire-codec", "peer-wire", "object-wire"] +members = [".", "runtime", "alloc", "crypto", "verbs", "gpu", "http", "topology", "telemetry", "flow", "control-wire", "controlplane", "uds-endpoint", "wire-codec", "peer-wire", "object-wire"] resolver = "3" [package] @@ -14,6 +14,8 @@ description = "Racer node-local read-through cache dataplane" default = [] # Enables the separately loaded native verbs adapter; unavailable hardware uses HTTP. rdma = ["rdma-verbs/native"] +# Enables the separately loaded CUDA shim (libracer_gpu_cuda.so.1); without a GPU, gpu-read is unavailable. +cuda = ["racer-gpu/native"] # Test-only Go SDK launcher; keeps crate-private fixture assembly out of default builds. subscription-interop = [] @@ -36,6 +38,7 @@ telemetry = { path = "telemetry" } http1 = { path = "http" } racer-crypto = { path = "crypto" } rdma-verbs = { path = "verbs" } +racer-gpu = { path = "gpu" } page-alloc = { path = "alloc" } uring-runtime = { path = "runtime" } base64 = "0.22" @@ -61,6 +64,7 @@ uds-endpoint = { path = "uds-endpoint", features = ["test-util", "simulation"] } flow-control = { path = "flow", features = ["simulation"] } wire-codec = { path = "wire-codec", features = ["simulation"] } rdma-verbs = { path = "verbs", features = ["simulation"] } +racer-gpu = { path = "gpu", features = ["simulation"] } http1 = { path = "http", features = ["test-util"] } page-alloc = { path = "alloc", features = ["simulation"] } uring-runtime = { path = "runtime", features = ["simulation", "test-util"] } diff --git a/cmd/racer-dataplane/alloc/src/slab.rs b/cmd/racer-dataplane/alloc/src/slab.rs index fb0965b709..4f1d7145cf 100644 --- a/cmd/racer-dataplane/alloc/src/slab.rs +++ b/cmd/racer-dataplane/alloc/src/slab.rs @@ -513,15 +513,25 @@ impl Slab { buffer: &AlignedBuffer, lease: &SegmentLease, ) -> Result<(Rc, Extent)> { - self.locate(extent, lease, |alignment, extent| { + self.locate_checked(extent, lease, |alignment, extent| { alignment.check(extent, buffer) }) } + /// Translate a leased logical extent to the worker-local descriptor and the + /// physical byte range on it, without submitting I/O or checking a buffer. + /// File backings return the logical extent unchanged; device backings add + /// the segment's placement offset. Callers that bypass io_uring (for + /// example cuFile) must keep the lease alive for the whole transfer, as + /// the leased generation is what keeps the bytes from being recycled. + pub fn locate(&self, extent: Extent, lease: &SegmentLease) -> Result<(Rc, Extent)> { + self.locate_checked(extent, lease, |_, _| Ok(())) + } + /// Validate a request's lease and extent, resolving its physical destination. /// `check` validates the transfer memory against both the logical and the /// physical extent with the slab's alignment. - fn locate( + fn locate_checked( &self, extent: Extent, lease: &SegmentLease, @@ -676,7 +686,7 @@ impl Slab { let (file, extent) = { let body = body.write_bytes()?; let header = header.as_slice(); - self.locate(extent, &lease, |alignment, extent| { + self.locate_checked(extent, &lease, |alignment, extent| { check_vectored(alignment, extent, header, body) })? }; @@ -1507,11 +1517,24 @@ mod tests { .unwrap(); let segments = Segments::new(4096); let _ = slab.open_configured(&segments).unwrap(); - drop(segments.append(4096).unwrap()); + let (first, first_extent) = segments.append(4096).unwrap(); let (lease, extent) = segments.append(4096).unwrap(); let buffer = slab.allocate(4096, ()).unwrap(); let (_, physical) = slab.submission(extent, &buffer, &lease).unwrap(); assert_eq!(physical, Extent::new(0, 4096).unwrap()); + // locate performs the same translation without a buffer. + let (descriptor, located) = slab.locate(extent, &lease).unwrap(); + assert_eq!(located, physical); + assert!(std::os::fd::AsRawFd::as_raw_fd(descriptor.as_ref()) >= 0); + assert_eq!( + slab.locate(first_extent, &first).unwrap().1, + Extent::new(8192, 4096).unwrap() + ); + drop(first); + assert!(matches!( + slab.locate(Extent::new(8192, 4096).unwrap(), &lease), + Err(Error::Corrupt) + )); assert!(matches!( slab.submission(Extent::new(0, 4096).unwrap(), &buffer, &lease), Err(Error::Corrupt) @@ -1546,6 +1569,13 @@ mod tests { }; placements[1].offset = offset; drop(opened); + // Translation itself does not enforce buffer alignment. + if offset == 1 { + assert_eq!( + slab.locate(extent, &lease).unwrap().1, + Extent::new(1, 4096).unwrap() + ); + } assert!( matches!(slab.submission(extent, &buffer, &lease), Err(error) if error == expected) ); diff --git a/cmd/racer-dataplane/crypto/src/lib.rs b/cmd/racer-dataplane/crypto/src/lib.rs index f38e3b726f..8570f6241d 100644 --- a/cmd/racer-dataplane/crypto/src/lib.rs +++ b/cmd/racer-dataplane/crypto/src/lib.rs @@ -153,7 +153,7 @@ impl PageCipher { } /// Derive the per-nonce AES-256-GCM key Kx from the first 12 nonce bytes. - fn derive(&self, nonce: &[u8; 24]) -> Result { + fn derive_key(&self, nonce: &[u8; 24]) -> Result, Error> { use aes::cipher::BlockCipherEncrypt; let mut kx = Zeroizing::new([0u8; 32]); for (index, half) in kx.chunks_exact_mut(16).enumerate() { @@ -166,6 +166,11 @@ impl PageCipher { self.aes .encrypt_block(half.try_into().map_err(|_| Error(()))?); } + Ok(kx) + } + + fn derive(&self, nonce: &[u8; 24]) -> Result { + let kx = self.derive_key(nonce)?; let key = ring::aead::UnboundKey::new(&ring::aead::AES_256_GCM, &kx[..]) .map_err(|_| Error(()))?; Ok(ring::aead::LessSafeKey::new(key)) @@ -320,6 +325,45 @@ where .map_err(|_| Error(())) } +/// The AES-256-GCM key and 96-bit nonce that XAES-256-GCM uses for one +/// 24-byte nonce. Lets a page be decrypted by a plain AES-256-GCM +/// implementation, such as the GPU kernels. The key is wiped on drop. +/// Intentionally does not implement Debug or Clone. +pub struct DerivedPageKey { + key: Zeroizing<[u8; 32]>, + + nonce: [u8; 12], +} + +impl DerivedPageKey { + /// The derived AES-256 key. Callers must not persist it. + pub fn key(&self) -> &[u8; 32] { + &self.key + } + + /// The 12-byte AES-GCM nonce: the last 12 bytes of the XAES nonce. + pub fn nonce(&self) -> &[u8; 12] { + &self.nonce + } +} + +/// Derive the AES-256-GCM key and nonce XAES-256-GCM uses for `nonce`, as +/// specified at : +/// `L = AES-256(K, 0^128)`, `K1 = L << 1` (xor `0x87` on carry), and the key is +/// `AES-256(K, [0,1,'X',0] || N[..12] ^ K1) || AES-256(K, [0,2,'X',0] || N[..12] ^ K1)`. +/// Shares the derivation [`seal`] and [`open`] use, so the result matches them +/// and `xaes_256_gcm::Xaes256Gcm` 0.1 byte for byte (see the tests). +pub fn xaes_derive(key: &[u8; 32], nonce: &[u8; 24]) -> DerivedPageKey { + let derived = PageCipher::new(key) + .derive_key(nonce) + .expect("16-byte halves always convert to AES blocks"); + let gcm_nonce: [u8; 12] = std::array::from_fn(|i| nonce[12 + i]); + DerivedPageKey { + key: derived, + nonce: gcm_nonce, + } +} + /// Seal the unchanged XChaCha20-Poly1305 credential format, not the page format. /// The caller must never reuse a nonce with the same key. pub fn seal_credentials( @@ -1084,6 +1128,20 @@ pub mod identity { .map_err(|_| Error::CorruptRecord) } + /// Derive the per-page AES-256-GCM key and nonce for offloaded page + /// decryption (the GPU path). Checks cache, key ID and purpose like + /// [`Self::open_page`]. Never exposes the epoch key itself; the + /// derived key is valid for this one XAES nonce and wiped on drop. + pub fn derive_page_subkey( + &self, + cache: &CacheId, + id: KeyId, + nonce: &[u8; 24], + ) -> Result { + let material = self.bound_material(cache, id, KeyPurpose::Page)?; + Ok(crate::xaes_derive(material, nonce)) + } + /// Seal credentials using only a credential epoch. Caller owns nonce and AAD. pub fn seal_credentials( &self, diff --git a/cmd/racer-dataplane/crypto/tests/primitives.rs b/cmd/racer-dataplane/crypto/tests/primitives.rs index 46d110b36c..437b4495c4 100644 --- a/cmd/racer-dataplane/crypto/tests/primitives.rs +++ b/cmd/racer-dataplane/crypto/tests/primitives.rs @@ -918,6 +918,33 @@ mod identity_workflows { } } + /// Offload derivation equals the primitive and enforces the lease binding. + #[test] + fn derived_page_subkey_matches_primitive_and_enforces_binding() { + let (_, _, roots) = issued_with(|_| {}); + let keys = keyring(roots); + let cache = CacheId(CACHE.into()); + let page = keys.active(&cache, KeyPurpose::Page).unwrap(); + let credentials = keys.active(&cache, KeyPurpose::OriginCredentials).unwrap(); + let nonce = *b"abcdefghijklMNOPQRSTUVWX"; + let derived = page.derive_page_subkey(&cache, page.id(), &nonce).unwrap(); + let expected = racer_crypto::xaes_derive(&[7; 32], &nonce); + assert_eq!(derived.key(), expected.key()); + assert_eq!(derived.nonce(), b"MNOPQRSTUVWX"); + assert!(matches!( + page.derive_page_subkey(&cache, credentials.id(), &nonce), + Err(Error::MissingKey) + )); + assert!(matches!( + page.derive_page_subkey(&CacheId("wrong".into()), page.id(), &nonce), + Err(Error::MissingKey) + )); + assert!(matches!( + credentials.derive_page_subkey(&cache, credentials.id(), &nonce), + Err(Error::MissingKey) + )); + } + /// Page and credential operations enforce bindings without touching failed output. #[test] fn borrowed_outputs_enforce_purpose_cache_id_and_retained_epoch() { @@ -1348,3 +1375,44 @@ fn xaes_tamper_zeroes_output_and_length_errors_do_not() { assert!(seal(&key, &nonce, aad, &plaintext, &mut long).is_err()); assert!(long.iter().all(|byte| *byte == 0xa5)); } + +/// The XAES derivation lets plain AES-256-GCM open XAES-256-GCM pages, and it +/// matches the upstream implementation for nonce halves and key bytes. +#[test] +fn xaes_derivation_matches_upstream_xaes_256_gcm() { + use xaes_256_gcm::aes_gcm::{ + Aes256Gcm, KeyInit, + aead::{AeadInOut, inout::InOutBuf}, + }; + + let mut state = 0x9e37_79b9_7f4a_7c15u64; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state as u8 + }; + for round in 0..64 { + let key: [u8; 32] = std::array::from_fn(|_| next()); + let nonce: [u8; 24] = std::array::from_fn(|_| next()); + let plaintext: Vec = (0..round * 7).map(|_| next()).collect(); + let aad: Vec = (0..round % 40).map(|_| next()).collect(); + let mut sealed = vec![0; plaintext.len() + TAG_LEN]; + seal(&key, &nonce, &aad, &plaintext, &mut sealed).unwrap(); + + let derived = racer_crypto::xaes_derive(&key, &nonce); + assert_eq!(derived.nonce(), &nonce[12..]); + let cipher = Aes256Gcm::new(derived.key().into()); + let (ciphertext, tag) = sealed.split_at(plaintext.len()); + let mut out = vec![0; plaintext.len()]; + cipher + .decrypt_inout_detached( + derived.nonce().into(), + &aad, + InOutBuf::new(ciphertext, &mut out).unwrap(), + tag.try_into().unwrap(), + ) + .unwrap(); + assert_eq!(out, plaintext); + } +} diff --git a/cmd/racer-dataplane/gpu/Cargo.toml b/cmd/racer-dataplane/gpu/Cargo.toml new file mode 100644 index 0000000000..12c8c78ab0 --- /dev/null +++ b/cmd/racer-dataplane/gpu/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "racer-gpu" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" +publish = false +description = "CUDA GPU discovery, client allocation import, staging, and GPU page decryption" + +[features] +default = [] +# Load libracer_gpu_cuda.so.1 (native/gpu_cuda.c) at runtime. +native = [] +# In-process fake GPUs backed by host memory, for tests without hardware. +simulation = [] + +[dependencies] +libc = "0.2" +zeroize = "1" + +[build-dependencies] +cc = "1" + +[dev-dependencies] +racer-crypto = { path = "../crypto" } diff --git a/cmd/racer-dataplane/gpu/build.rs b/cmd/racer-dataplane/gpu/build.rs new file mode 100644 index 0000000000..44fd1d1639 --- /dev/null +++ b/cmd/racer-dataplane/gpu/build.rs @@ -0,0 +1,17 @@ +//! Compile the host emulation of the GPU decrypt kernels (`native/gcm_host.c`, +//! which includes `native/gcm_core.h`) into a static library. Only the +//! simulation backend and tests reference it, so release builds without them +//! link none of it. `libracer_gpu_cuda.so.1` is built separately by +//! `images/racer-dataplane/build-native-cuda.sh`. +fn main() { + println!("cargo:rerun-if-changed=native/gcm_core.h"); + println!("cargo:rerun-if-changed=native/gcm_host.c"); + cc::Build::new() + .file("native/gcm_host.c") + .std("c99") + .opt_level(3) + .warnings(true) + .extra_warnings(true) + .warnings_into_errors(true) + .compile("racer_gpu_gcm_host"); +} diff --git a/cmd/racer-dataplane/gpu/native/cuda_private.h b/cmd/racer-dataplane/gpu/native/cuda_private.h new file mode 100644 index 0000000000..cf85676cd4 --- /dev/null +++ b/cmd/racer-dataplane/gpu/native/cuda_private.h @@ -0,0 +1,194 @@ +/* The few CUDA driver, NVML and cuFile declarations gpu_cuda.c needs. + * + * The shim loads every library with dlopen, so it builds without the CUDA + * toolkit. Values and layouts are copied from the vendor headers: + * + * cuda.h (CUDA 12.9, CUDA_VERSION 12090): + * https://gitlab.com/nvidia/headers/cuda-individual/cudart/-/raw/main/cuda.h + * https://docs.nvidia.com/cuda/cuda-driver-api/ + * nvml.h: + * https://gitlab.com/nvidia/headers/cuda-individual/nvml_dev/-/raw/main/nvml.h + * https://docs.nvidia.com/deploy/nvml-api/ + * cufile.h (GDS cuFile API reference, section 4.1): + * https://docs.nvidia.com/gpudirect-storage/api-reference-guide/index.html + * + * Symbol names include the version suffix cuda.h maps the plain name to (for + * example cuMemAlloc -> cuMemAlloc_v2), because dlsym sees the real export. + * Only append to this file; never change a value without the vendor header. */ +#ifndef RACER_GPU_CUDA_PRIVATE_H +#define RACER_GPU_CUDA_PRIVATE_H + +#include +#include +#include +#include + +/* ---- CUDA driver API (cuda.h) ---- */ +typedef int CUresult; /* enum cudaError_enum; CUDA_SUCCESS = 0 */ +typedef int CUdevice; /* CUdevice_v1 */ +typedef unsigned long long CUdeviceptr; /* CUdeviceptr_v2 on 64-bit hosts */ +typedef unsigned long long CUmemGenericAllocationHandle; +typedef struct CUctx_st *CUcontext; +typedef struct CUstream_st *CUstream; +typedef struct CUmod_st *CUmodule; +typedef struct CUfunc_st *CUfunction; +typedef struct { char bytes[16]; } CUuuid; + +#define CUDA_SUCCESS 0 + +/* CUdevice_attribute */ +#define CU_DEVICE_ATTRIBUTE_COMPUTE_MODE 20 +#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED 102 +#define CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR_SUPPORTED 103 +#define CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_SUPPORTED 116 +#define CU_DEVICE_ATTRIBUTE_DMA_BUF_SUPPORTED 124 + +/* CUstream_flags */ +#define CU_STREAM_NON_BLOCKING 0x1 +/* CUmemAllocationHandleType */ +#define CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR 0x1 +/* CUmemAccess_flags */ +#define CU_MEM_ACCESS_FLAGS_PROT_READWRITE 0x3 +/* CUmemLocationType */ +#define CU_MEM_LOCATION_TYPE_DEVICE 0x1 +/* CUmemAllocationType */ +#define CU_MEM_ALLOCATION_TYPE_PINNED 0x1 +/* CUmemAllocationGranularity_flags */ +#define CU_MEM_ALLOC_GRANULARITY_MINIMUM 0x0 +/* CUmemRangeHandleType */ +#define CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD 0x1 + +typedef struct { + int type; /* CUmemLocationType */ + int id; +} CUmemLocation; + +typedef struct { + int type; /* CUmemAllocationType */ + int requestedHandleTypes; /* CUmemAllocationHandleType */ + CUmemLocation location; + void *win32HandleMetaData; + struct { + unsigned char compressionType; + unsigned char gpuDirectRDMACapable; + unsigned short usage; + unsigned char reserved[4]; + } allocFlags; +} CUmemAllocationProp; + +typedef struct { + CUmemLocation location; + int flags; /* CUmemAccess_flags */ +} CUmemAccessDesc; + +_Static_assert(sizeof(CUmemLocation) == 8, "CUmemLocation layout"); +_Static_assert(sizeof(CUmemAllocationProp) == 32, "CUmemAllocationProp layout"); +_Static_assert(offsetof(CUmemAllocationProp, win32HandleMetaData) == 16, "CUmemAllocationProp layout"); +_Static_assert(sizeof(CUmemAccessDesc) == 12, "CUmemAccessDesc layout"); + +/* X(name, symbol, return, arguments) for every driver function. */ +#define RACER_CUDA_FUNCTIONS(X) \ + X(cuInit, "cuInit", CUresult, (unsigned int flags)) \ + X(cuDeviceGetCount, "cuDeviceGetCount", CUresult, (int *count)) \ + X(cuDeviceGet, "cuDeviceGet", CUresult, (CUdevice *device, int ordinal)) \ + X(cuDeviceGetUuid, "cuDeviceGetUuid_v2", CUresult, (CUuuid *uuid, CUdevice dev)) \ + X(cuDeviceGetPCIBusId, "cuDeviceGetPCIBusId", CUresult, (char *id, int len, CUdevice dev)) \ + X(cuDeviceGetAttribute, "cuDeviceGetAttribute", CUresult, (int *pi, int attrib, CUdevice dev)) \ + X(cuDeviceTotalMem, "cuDeviceTotalMem_v2", CUresult, (size_t *bytes, CUdevice dev)) \ + X(cuDevicePrimaryCtxRetain, "cuDevicePrimaryCtxRetain", CUresult, (CUcontext *ctx, CUdevice dev)) \ + X(cuDevicePrimaryCtxRelease, "cuDevicePrimaryCtxRelease_v2", CUresult, (CUdevice dev)) \ + X(cuCtxPushCurrent, "cuCtxPushCurrent_v2", CUresult, (CUcontext ctx)) \ + X(cuCtxPopCurrent, "cuCtxPopCurrent_v2", CUresult, (CUcontext *ctx)) \ + X(cuMemImportFromShareableHandle, "cuMemImportFromShareableHandle", CUresult, \ + (CUmemGenericAllocationHandle *handle, void *os_handle, int type)) \ + X(cuMemGetAllocationPropertiesFromHandle, "cuMemGetAllocationPropertiesFromHandle", CUresult, \ + (CUmemAllocationProp *prop, CUmemGenericAllocationHandle handle)) \ + X(cuMemGetAllocationGranularity, "cuMemGetAllocationGranularity", CUresult, \ + (size_t *granularity, const CUmemAllocationProp *prop, int option)) \ + X(cuMemAddressReserve, "cuMemAddressReserve", CUresult, \ + (CUdeviceptr *ptr, size_t size, size_t alignment, CUdeviceptr addr, unsigned long long flags)) \ + X(cuMemAddressFree, "cuMemAddressFree", CUresult, (CUdeviceptr ptr, size_t size)) \ + X(cuMemMap, "cuMemMap", CUresult, \ + (CUdeviceptr ptr, size_t size, size_t offset, CUmemGenericAllocationHandle handle, \ + unsigned long long flags)) \ + X(cuMemUnmap, "cuMemUnmap", CUresult, (CUdeviceptr ptr, size_t size)) \ + X(cuMemSetAccess, "cuMemSetAccess", CUresult, \ + (CUdeviceptr ptr, size_t size, const CUmemAccessDesc *desc, size_t count)) \ + X(cuMemRelease, "cuMemRelease", CUresult, (CUmemGenericAllocationHandle handle)) \ + X(cuMemAlloc, "cuMemAlloc_v2", CUresult, (CUdeviceptr *ptr, size_t size)) \ + X(cuMemFree, "cuMemFree_v2", CUresult, (CUdeviceptr ptr)) \ + X(cuMemAllocHost, "cuMemAllocHost_v2", CUresult, (void **ptr, size_t size)) \ + X(cuMemFreeHost, "cuMemFreeHost", CUresult, (void *ptr)) \ + X(cuMemcpyHtoDAsync, "cuMemcpyHtoDAsync_v2", CUresult, \ + (CUdeviceptr dst, const void *src, size_t n, CUstream stream)) \ + X(cuMemcpyDtoDAsync, "cuMemcpyDtoDAsync_v2", CUresult, \ + (CUdeviceptr dst, CUdeviceptr src, size_t n, CUstream stream)) \ + X(cuMemcpyDtoHAsync, "cuMemcpyDtoHAsync_v2", CUresult, \ + (void *dst, CUdeviceptr src, size_t n, CUstream stream)) \ + X(cuMemsetD8Async, "cuMemsetD8Async", CUresult, \ + (CUdeviceptr dst, unsigned char value, size_t n, CUstream stream)) \ + X(cuStreamCreate, "cuStreamCreate", CUresult, (CUstream *stream, unsigned int flags)) \ + X(cuStreamSynchronize, "cuStreamSynchronize", CUresult, (CUstream stream)) \ + X(cuStreamDestroy, "cuStreamDestroy_v2", CUresult, (CUstream stream)) \ + X(cuModuleLoadData, "cuModuleLoadData", CUresult, (CUmodule *module, const void *image)) \ + X(cuModuleUnload, "cuModuleUnload", CUresult, (CUmodule module)) \ + X(cuModuleGetFunction, "cuModuleGetFunction", CUresult, \ + (CUfunction *fn, CUmodule module, const char *name)) \ + X(cuLaunchKernel, "cuLaunchKernel", CUresult, \ + (CUfunction f, unsigned int gx, unsigned int gy, unsigned int gz, unsigned int bx, \ + unsigned int by, unsigned int bz, unsigned int shared, CUstream stream, void **params, \ + void **extra)) \ + X(cuMemGetHandleForAddressRange, "cuMemGetHandleForAddressRange", CUresult, \ + (void *handle, CUdeviceptr ptr, size_t size, int type, unsigned long long flags)) + +/* ---- NVML (nvml.h), optional; used only for the NUMA node ---- */ +typedef int nvmlReturn_t; /* NVML_SUCCESS = 0 */ +typedef struct nvmlDevice_st *nvmlDevice_t; +#define NVML_SUCCESS 0 + +#define RACER_NVML_FUNCTIONS(X) \ + X(nvmlInit, "nvmlInit_v2", nvmlReturn_t, (void)) \ + X(nvmlShutdown, "nvmlShutdown", nvmlReturn_t, (void)) \ + X(nvmlDeviceGetHandleByPciBusId, "nvmlDeviceGetHandleByPciBusId_v2", nvmlReturn_t, \ + (const char *id, nvmlDevice_t *device)) \ + X(nvmlDeviceGetNumaNodeId, "nvmlDeviceGetNumaNodeId", nvmlReturn_t, \ + (nvmlDevice_t device, unsigned int *node)) + +/* ---- cuFile (cufile.h), optional ---- */ +typedef struct { + int err; /* CUfileOpError; CU_FILE_SUCCESS = 0 */ + int cu_err; /* CUresult */ +} CUfileError_t; +typedef void *CUfileHandle_t; +typedef struct { + int type; /* CUfileFileHandleType */ + union { + int fd; + void *handle; + } handle; + const void *fs_ops; +} CUfileDescr_t; + +#define CU_FILE_SUCCESS 0 +#define CU_FILE_HANDLE_TYPE_OPAQUE_FD 1 + +_Static_assert(sizeof(CUfileError_t) == 8, "CUfileError_t layout"); +_Static_assert(sizeof(CUfileDescr_t) == 24, "CUfileDescr_t layout"); +_Static_assert(offsetof(CUfileDescr_t, handle) == 8, "CUfileDescr_t layout"); + +/* cuFileSetParameterBool (cuFile 1.10+); resolved optionally. */ +#define CUFILE_PARAM_PROPERTIES_ALLOW_COMPAT_MODE 1 +typedef CUfileError_t (*cuFileSetParameterBool_fn)(int param, bool value); + +#define RACER_CUFILE_FUNCTIONS(X) \ + X(cuFileDriverOpen, "cuFileDriverOpen", CUfileError_t, (void)) \ + X(cuFileHandleRegister, "cuFileHandleRegister", CUfileError_t, \ + (CUfileHandle_t *fh, CUfileDescr_t *descr)) \ + X(cuFileHandleDeregister, "cuFileHandleDeregister", void, (CUfileHandle_t fh)) \ + X(cuFileBufRegister, "cuFileBufRegister", CUfileError_t, \ + (const void *base, size_t size, int flags)) \ + X(cuFileBufDeregister, "cuFileBufDeregister", CUfileError_t, (const void *base)) \ + X(cuFileRead, "cuFileRead", ssize_t, \ + (CUfileHandle_t fh, void *base, size_t size, off_t file_offset, off_t base_offset)) + +#endif diff --git a/cmd/racer-dataplane/gpu/native/gcm_core.h b/cmd/racer-dataplane/gpu/native/gcm_core.h new file mode 100644 index 0000000000..048a9d3c49 --- /dev/null +++ b/cmd/racer-dataplane/gpu/native/gcm_core.h @@ -0,0 +1,342 @@ +/* AES-256-GCM decrypt-with-verification and CRC-64/XZ, shared by the CUDA + * kernels (gcm_kernel.cu), the host shim (gpu_cuda.c), the simulation backend + * and the CPU tests (gcm_host.c). It compiles as C99 and as CUDA. + * + * Racer pages are sealed with XAES-256-GCM (https://c2sp.org/XAES-256-GCM). + * The CPU derives the per-page AES-256 key and 12-byte GCM nonce, so this file + * only implements plain AES-256-GCM (NIST SP 800-38D) on the derived values. + * The sealed page is ciphertext followed by the 16-byte tag; the stored + * checksum is CRC-64/XZ over the whole sealed page (racer_crypto::crc64 over + * CiphertextPage::bytes()). + * + * The work is split so it runs in parallel: + * + * gcm_prepare host: key schedule, H, E(K, J0), and the GHASH terms + * that do not depend on the ciphertext (AAD, lengths). + * gcm_chunk_partial one per GCM_CHUNK bytes: CRC of the chunk shifted to + * its final position, and the chunk's GHASH sum + * multiplied by the power of H it has in the full sum. + * gcm_finalize XOR of all partials (both sums are linear), then the + * constant-time tag compare and the CRC compare. + * gcm_ctr_block one per 16-byte block: CTR decryption. + * + * GHASH is sum(X_i * H^(m-i+1)) over m blocks, so chunk j with last block + * b_j contributes Horner(chunk) * H^(nc-b_j+1), where nc is the number of + * ciphertext blocks. CRC-64/XZ composes like zlib's crc32_combine: + * crc(A||B) = crc(A) * x^(8|B|) mod P xor crc(B), so each chunk is shifted by + * the number of bytes after it and all chunks are XORed. + * + * Timing: GF(2^128) multiplication is constant time (no secret-dependent + * branches or indexes). AES uses an S-box lookup with secret indexes, so it is + * not constant time with respect to cache or shared-memory bank timing. The + * kernels keep the S-box in shared memory, where only bank conflicts (not a + * cache hierarchy shared with other tenants) can leak timing, and the key is a + * per-page derived key. A bitsliced AES would remove this; it is future work. + */ +#ifndef RACER_GCM_CORE_H +#define RACER_GCM_CORE_H + +#include +#include + +#ifdef __CUDACC__ +#define GCM_FN __host__ __device__ static inline +#else +#define GCM_FN static inline +#endif + +/* Bytes of sealed input each partial covers. A multiple of 16. */ +#define GCM_CHUNK 4096u +/* Reflected CRC-64/XZ (ECMA-182) polynomial. */ +#define GCM_CRC64_POLY 0xC96C5795D7870F42ull +/* Verdict values. Only GCM_VERDICT_PASS lets CTR decryption write output. */ +#define GCM_VERDICT_NONE 0u +#define GCM_VERDICT_PASS 0x600Du +#define GCM_VERDICT_CRC 2u +#define GCM_VERDICT_TAG 3u + +/* FIPS-197 S-box. */ +#define GCM_SBOX_INIT { \ + 0x63,0x7c,0x77,0x7b,0xf2,0x6b,0x6f,0xc5,0x30,0x01,0x67,0x2b,0xfe,0xd7,0xab,0x76, \ + 0xca,0x82,0xc9,0x7d,0xfa,0x59,0x47,0xf0,0xad,0xd4,0xa2,0xaf,0x9c,0xa4,0x72,0xc0, \ + 0xb7,0xfd,0x93,0x26,0x36,0x3f,0xf7,0xcc,0x34,0xa5,0xe5,0xf1,0x71,0xd8,0x31,0x15, \ + 0x04,0xc7,0x23,0xc3,0x18,0x96,0x05,0x9a,0x07,0x12,0x80,0xe2,0xeb,0x27,0xb2,0x75, \ + 0x09,0x83,0x2c,0x1a,0x1b,0x6e,0x5a,0xa0,0x52,0x3b,0xd6,0xb3,0x29,0xe3,0x2f,0x84, \ + 0x53,0xd1,0x00,0xed,0x20,0xfc,0xb1,0x5b,0x6a,0xcb,0xbe,0x39,0x4a,0x4c,0x58,0xcf, \ + 0xd0,0xef,0xaa,0xfb,0x43,0x4d,0x33,0x85,0x45,0xf9,0x02,0x7f,0x50,0x3c,0x9f,0xa8, \ + 0x51,0xa3,0x40,0x8f,0x92,0x9d,0x38,0xf5,0xbc,0xb6,0xda,0x21,0x10,0xff,0xf3,0xd2, \ + 0xcd,0x0c,0x13,0xec,0x5f,0x97,0x44,0x17,0xc4,0xa7,0x7e,0x3d,0x64,0x5d,0x19,0x73, \ + 0x60,0x81,0x4f,0xdc,0x22,0x2a,0x90,0x88,0x46,0xee,0xb8,0x14,0xde,0x5e,0x0b,0xdb, \ + 0xe0,0x32,0x3a,0x0a,0x49,0x06,0x24,0x5c,0xc2,0xd3,0xac,0x62,0x91,0x95,0xe4,0x79, \ + 0xe7,0xc8,0x37,0x6d,0x8d,0xd5,0x4e,0xa9,0x6c,0x56,0xf4,0xea,0x65,0x7a,0xae,0x08, \ + 0xba,0x78,0x25,0x2e,0x1c,0xa6,0xb4,0xc6,0xe8,0xdd,0x74,0x1f,0x4b,0xbd,0x8b,0x8a, \ + 0x70,0x3e,0xb5,0x66,0x48,0x03,0xf6,0x0e,0x61,0x35,0x57,0xb9,0x86,0xc1,0x1d,0x9e, \ + 0xe1,0xf8,0x98,0x11,0x69,0xd9,0x8e,0x94,0x9b,0x1e,0x87,0xe9,0xce,0x55,0x28,0xdf, \ + 0x8c,0xa1,0x89,0x0d,0xbf,0xe6,0x42,0x68,0x41,0x99,0x2d,0x0f,0xb0,0x54,0xbb,0x16 } + +/* Everything the device needs for one page. Wiped after use. */ +struct gcm_key { + uint8_t rk[240]; /* AES-256 round keys */ + uint8_t h[16]; /* E(K, 0^128) */ + uint8_t ekj0[16]; /* E(K, J0), the tag mask */ + uint8_t prefix[16]; /* GHASH terms for AAD and the length block */ + uint8_t nonce[12]; + uint32_t reserved; +}; + +/* One chunk's contribution. */ +struct gcm_partial { + uint8_t ghash[16]; + uint64_t crc; + uint64_t reserved; +}; + +/* Result of verification. Same layout as struct racer_gpu_verdict. */ +struct gcm_verdict { + uint32_t status; + uint32_t reserved; + uint64_t crc; +}; + +/* GF(2^128) element in GCM bit order, as two big-endian halves. */ +struct gcm_block { + uint64_t hi, lo; +}; + +GCM_FN uint64_t gcm_load64(const uint8_t *p) { + uint64_t v = 0; + for (int i = 0; i < 8; ++i) v = (v << 8) | p[i]; + return v; +} + +GCM_FN void gcm_store64(uint8_t *p, uint64_t v) { + for (int i = 7; i >= 0; --i) { p[i] = (uint8_t)v; v >>= 8; } +} + +GCM_FN struct gcm_block gcm_load(const uint8_t *p) { + struct gcm_block b; + b.hi = gcm_load64(p); + b.lo = gcm_load64(p + 8); + return b; +} + +/* Load up to 16 bytes, zero padding the rest. */ +GCM_FN struct gcm_block gcm_load_partial(const uint8_t *p, uint64_t n) { + uint8_t tmp[16]; + for (unsigned i = 0; i < 16; ++i) tmp[i] = i < n ? p[i] : 0; + return gcm_load(tmp); +} + +GCM_FN void gcm_store(uint8_t *p, struct gcm_block b) { + gcm_store64(p, b.hi); + gcm_store64(p + 8, b.lo); +} + +/* X * Y in GF(2^128), constant time (SP 800-38D algorithm 1). */ +GCM_FN struct gcm_block gcm_mul(struct gcm_block x, struct gcm_block y) { + struct gcm_block z = {0, 0}, v = y; + for (int i = 0; i < 128; ++i) { + uint64_t bit = i < 64 ? (x.hi >> (63 - i)) & 1 : (x.lo >> (127 - i)) & 1; + uint64_t mask = (uint64_t)0 - bit; + z.hi ^= v.hi & mask; + z.lo ^= v.lo & mask; + uint64_t lsb = (uint64_t)0 - (v.lo & 1); + v.lo = (v.lo >> 1) | (v.hi << 63); + v.hi = (v.hi >> 1) ^ (0xE100000000000000ull & lsb); + } + return z; +} + +/* H^k. k is public; the multiplications are constant time in H. */ +GCM_FN struct gcm_block gcm_pow(struct gcm_block h, uint64_t k) { + struct gcm_block r = {0x8000000000000000ull, 0}; /* multiplicative one */ + while (k) { + if (k & 1) r = gcm_mul(r, h); + h = gcm_mul(h, h); + k >>= 1; + } + return r; +} + +GCM_FN uint8_t gcm_xtime(uint8_t x) { + return (uint8_t)((x << 1) ^ (0x1b & (uint8_t)(0u - (x >> 7)))); +} + +/* FIPS-197 AES-256 key expansion into 15 round keys. */ +GCM_FN void gcm_aes256_expand(const uint8_t *sbox, const uint8_t key[32], uint8_t rk[240]) { + uint8_t rcon = 1; + for (int i = 0; i < 32; ++i) rk[i] = key[i]; + for (int i = 8; i < 60; ++i) { + uint8_t t[4]; + for (int j = 0; j < 4; ++j) t[j] = rk[4 * (i - 1) + j]; + if (i % 8 == 0) { + uint8_t first = t[0]; + t[0] = (uint8_t)(sbox[t[1]] ^ rcon); + t[1] = sbox[t[2]]; + t[2] = sbox[t[3]]; + t[3] = sbox[first]; + rcon = gcm_xtime(rcon); + } else if (i % 8 == 4) { + for (int j = 0; j < 4; ++j) t[j] = sbox[t[j]]; + } + for (int j = 0; j < 4; ++j) rk[4 * i + j] = (uint8_t)(rk[4 * (i - 8) + j] ^ t[j]); + } +} + +/* Encrypt one block with AES-256 round keys `rk`. */ +GCM_FN void gcm_aes256_encrypt(const uint8_t *sbox, const uint8_t *rk, const uint8_t in[16], + uint8_t out[16]) { + uint8_t s[16], t[16]; + for (int i = 0; i < 16; ++i) s[i] = (uint8_t)(in[i] ^ rk[i]); + for (int round = 1; round <= 14; ++round) { + /* SubBytes and ShiftRows. Byte r + 4c is row r, column c. */ + for (int c = 0; c < 4; ++c) + for (int r = 0; r < 4; ++r) t[r + 4 * c] = sbox[s[r + 4 * ((c + r) & 3)]]; + if (round != 14) { + for (int c = 0; c < 4; ++c) { + uint8_t a0 = t[4 * c], a1 = t[4 * c + 1], a2 = t[4 * c + 2], a3 = t[4 * c + 3]; + uint8_t all = (uint8_t)(a0 ^ a1 ^ a2 ^ a3); + s[4 * c] = (uint8_t)(a0 ^ all ^ gcm_xtime((uint8_t)(a0 ^ a1))); + s[4 * c + 1] = (uint8_t)(a1 ^ all ^ gcm_xtime((uint8_t)(a1 ^ a2))); + s[4 * c + 2] = (uint8_t)(a2 ^ all ^ gcm_xtime((uint8_t)(a2 ^ a3))); + s[4 * c + 3] = (uint8_t)(a3 ^ all ^ gcm_xtime((uint8_t)(a3 ^ a0))); + } + } else { + for (int i = 0; i < 16; ++i) s[i] = t[i]; + } + for (int i = 0; i < 16; ++i) s[i] ^= rk[16 * round + i]; + } + for (int i = 0; i < 16; ++i) out[i] = s[i]; +} + +/* Entry `i` of the reflected CRC-64/XZ byte table. */ +GCM_FN uint64_t gcm_crc64_entry(unsigned i) { + uint64_t c = i; + for (int k = 0; k < 8; ++k) c = (c & 1) ? (c >> 1) ^ GCM_CRC64_POLY : c >> 1; + return c; +} + +/* Reflected CRC-64/XZ byte table. */ +GCM_FN void gcm_crc64_table(uint64_t table[256]) { + for (unsigned i = 0; i < 256; ++i) table[i] = gcm_crc64_entry(i); +} + +/* Complete CRC-64/XZ (init and xorout all ones) of `n` bytes. */ +GCM_FN uint64_t gcm_crc64(const uint64_t *table, const uint8_t *p, uint64_t n) { + uint64_t c = ~(uint64_t)0; + for (uint64_t i = 0; i < n; ++i) c = table[(c ^ p[i]) & 0xff] ^ (c >> 8); + return ~c; +} + +/* a * b mod P in the reflected representation (bit 63 is x^0). */ +GCM_FN uint64_t gcm_crc_mulmod(uint64_t a, uint64_t b) { + uint64_t m = 1ull << 63, p = 0; + if (!a) return 0; + for (;;) { + if (a & m) { + p ^= b; + if ((a & (m - 1)) == 0) break; + } + m >>= 1; + b = (b & 1) ? (b >> 1) ^ GCM_CRC64_POLY : b >> 1; + } + return p; +} + +/* crc * x^(8n) mod P: moves a complete CRC past n more bytes. */ +GCM_FN uint64_t gcm_crc_shift(uint64_t crc, uint64_t n) { + uint64_t p = 1ull << 63, base = 1ull << (63 - 8); + while (n) { + if (n & 1) p = gcm_crc_mulmod(base, p); + base = gcm_crc_mulmod(base, base); + n >>= 1; + } + return gcm_crc_mulmod(p, crc); +} + +/* Number of partials for a page of `n` plaintext bytes. */ +GCM_FN uint64_t gcm_chunks(uint64_t n) { + return (n + 16 + GCM_CHUNK - 1) / GCM_CHUNK; +} + +/* Host side: derive everything the kernels need from the derived key, + * nonce, AAD and plaintext length. */ +GCM_FN void gcm_prepare(const uint8_t *sbox, const uint8_t key[32], const uint8_t nonce[12], + const uint8_t *aad, uint64_t aad_len, uint64_t n, struct gcm_key *out) { + uint8_t block[16]; + gcm_aes256_expand(sbox, key, out->rk); + for (int i = 0; i < 16; ++i) block[i] = 0; + gcm_aes256_encrypt(sbox, out->rk, block, out->h); + for (int i = 0; i < 12; ++i) block[i] = out->nonce[i] = nonce[i]; + block[12] = 0; block[13] = 0; block[14] = 0; block[15] = 1; + gcm_aes256_encrypt(sbox, out->rk, block, out->ekj0); + out->reserved = 0; + + struct gcm_block h = gcm_load(out->h), y = {0, 0}; + for (uint64_t i = 0; i < aad_len; i += 16) { + struct gcm_block x = gcm_load_partial(aad + i, aad_len - i); + y.hi ^= x.hi; y.lo ^= x.lo; + y = gcm_mul(y, h); + } + uint64_t nc = (n + 15) / 16; + struct gcm_block prefix = gcm_mul(y, gcm_pow(h, nc + 1)); + struct gcm_block lengths = {aad_len * 8, n * 8}; + struct gcm_block lh = gcm_mul(lengths, h); + prefix.hi ^= lh.hi; prefix.lo ^= lh.lo; + gcm_store(out->prefix, prefix); + for (int i = 0; i < 16; ++i) block[i] = 0; +} + +/* Partial `j` over the sealed page (`n` ciphertext bytes then a 16-byte tag). */ +GCM_FN void gcm_chunk_partial(const uint64_t *crc_table, const struct gcm_key *key, + const uint8_t *sealed, uint64_t n, uint64_t j, + struct gcm_partial *out) { + uint64_t total = n + 16; + uint64_t start = j * GCM_CHUNK; + uint64_t end = start + GCM_CHUNK < total ? start + GCM_CHUNK : total; + out->crc = gcm_crc_shift(gcm_crc64(crc_table, sealed + start, end - start), total - end); + out->reserved = 0; + + struct gcm_block y = {0, 0}; + uint64_t gend = end < n ? end : n; + if (start < gend) { + struct gcm_block h = gcm_load(key->h); + for (uint64_t i = start; i < gend; i += 16) { + struct gcm_block x = gcm_load_partial(sealed + i, gend - i); + y.hi ^= x.hi; y.lo ^= x.lo; + y = gcm_mul(y, h); + } + uint64_t nc = (n + 15) / 16, last = (gend + 15) / 16; + y = gcm_mul(y, gcm_pow(h, nc - last + 1)); + } + gcm_store(out->ghash, y); +} + +/* Combine reduced partials, check the tag in constant time and the CRC. */ +GCM_FN void gcm_finalize(const struct gcm_key *key, const uint8_t *sealed, uint64_t n, + const uint8_t ghash[16], uint64_t crc, uint32_t check_crc, + uint64_t expected_crc, struct gcm_verdict *out) { + uint8_t diff = 0; + for (int i = 0; i < 16; ++i) + diff |= (uint8_t)(key->prefix[i] ^ ghash[i] ^ key->ekj0[i] ^ sealed[n + i]); + int crc_ok = !check_crc || crc == expected_crc; + out->crc = crc; + out->reserved = 0; + out->status = !crc_ok ? GCM_VERDICT_CRC : diff ? GCM_VERDICT_TAG : GCM_VERDICT_PASS; +} + +/* CTR-decrypt block `i` into `out`. Call only after a passing verdict. */ +GCM_FN void gcm_ctr_block(const uint8_t *sbox, const struct gcm_key *key, const uint8_t *sealed, + uint64_t n, uint64_t i, uint8_t *out) { + uint8_t counter[16], stream[16]; + uint32_t c = (uint32_t)(i + 2); + for (int k = 0; k < 12; ++k) counter[k] = key->nonce[k]; + counter[12] = (uint8_t)(c >> 24); counter[13] = (uint8_t)(c >> 16); + counter[14] = (uint8_t)(c >> 8); counter[15] = (uint8_t)c; + gcm_aes256_encrypt(sbox, key->rk, counter, stream); + uint64_t base = i * 16, len = n - base < 16 ? n - base : 16; + for (uint64_t k = 0; k < len; ++k) out[base + k] = (uint8_t)(sealed[base + k] ^ stream[k]); + for (int k = 0; k < 16; ++k) stream[k] = 0; +} + +#endif diff --git a/cmd/racer-dataplane/gpu/native/gcm_host.c b/cmd/racer-dataplane/gpu/native/gcm_host.c new file mode 100644 index 0000000000..e1064e5cce --- /dev/null +++ b/cmd/racer-dataplane/gpu/native/gcm_host.c @@ -0,0 +1,70 @@ +/* Serial host emulation of the GPU decrypt kernels. + * + * Runs the same gcm_core.h steps in the same order the kernels do: prepare, + * one partial per chunk, XOR reduction, finalize, then CTR only on a passing + * verdict. The CPU tests compare it with the racer-crypto crate, and the + * simulation backend uses it as its "kernel". Linked statically into the Rust + * crate by build.rs; never part of libracer_gpu_cuda.so.1. */ +#include "gcm_core.h" + +#include +#include + +static const uint8_t host_sbox[256] = GCM_SBOX_INIT; + +/* Wipe memory the compiler may not elide. */ +static void wipe(void *p, size_t n) { + volatile uint8_t *v = (volatile uint8_t *)p; + while (n--) *v++ = 0; +} + +/* CRC-64/XZ through the chunked combine path. */ +uint64_t racer_gcm_host_crc64(const uint8_t *data, uint64_t length) { + uint64_t table[256], crc = 0; + gcm_crc64_table(table); + if (!length) return gcm_crc64(table, data, 0); + for (uint64_t start = 0; start < length; start += GCM_CHUNK) { + uint64_t end = start + GCM_CHUNK < length ? start + GCM_CHUNK : length; + crc ^= gcm_crc_shift(gcm_crc64(table, data + start, end - start), length - end); + } + return crc; +} + +/* One AES-256 block, for known-answer tests. */ +void racer_gcm_host_aes256(const uint8_t key[32], const uint8_t in[16], uint8_t out[16]) { + uint8_t rk[240]; + gcm_aes256_expand(host_sbox, key, rk); + gcm_aes256_encrypt(host_sbox, rk, in, out); + wipe(rk, sizeof(rk)); +} + +/* Verify and decrypt `sealed` (n ciphertext bytes then a 16-byte tag) into + * `out` (n bytes). `out` is written only when the verdict passes. + * Returns 0, or -1 if scratch memory cannot be allocated. */ +int racer_gcm_host_decrypt(const uint8_t key[32], const uint8_t nonce[12], const uint8_t *aad, + uint64_t aad_len, const uint8_t *sealed, uint64_t n, + uint32_t check_crc, uint64_t expected_crc, uint8_t *out, + struct gcm_verdict *verdict) { + struct gcm_key k; + uint64_t table[256]; + uint64_t chunks = gcm_chunks(n); + struct gcm_partial *partials = calloc(chunks, sizeof(*partials)); + verdict->status = GCM_VERDICT_NONE; + if (!partials) return -1; + gcm_crc64_table(table); + gcm_prepare(host_sbox, key, nonce, aad, aad_len, n, &k); + for (uint64_t j = 0; j < chunks; ++j) gcm_chunk_partial(table, &k, sealed, n, j, &partials[j]); + uint8_t ghash[16] = {0}; + uint64_t crc = 0; + for (uint64_t j = 0; j < chunks; ++j) { + for (int i = 0; i < 16; ++i) ghash[i] ^= partials[j].ghash[i]; + crc ^= partials[j].crc; + } + gcm_finalize(&k, sealed, n, ghash, crc, check_crc, expected_crc, verdict); + if (verdict->status == GCM_VERDICT_PASS) + for (uint64_t i = 0; i < (n + 15) / 16; ++i) gcm_ctr_block(host_sbox, &k, sealed, n, i, out); + wipe(&k, sizeof(k)); + wipe(partials, chunks * sizeof(*partials)); + free(partials); + return 0; +} diff --git a/cmd/racer-dataplane/gpu/native/gcm_kernel.cu b/cmd/racer-dataplane/gpu/native/gcm_kernel.cu new file mode 100644 index 0000000000..35cb78fcc1 --- /dev/null +++ b/cmd/racer-dataplane/gpu/native/gcm_kernel.cu @@ -0,0 +1,81 @@ +/* CUDA kernels for GPU page decryption. See gcm_core.h for the math. + * + * Built to PTX by images/racer-dataplane/build-native-cuda.sh with + * `nvcc -ptx -arch=compute_80`. compute_80 (Ampere) is the baseline: the + * driver JIT-compiles the PTX for the installed GPU, so newer GPUs work and + * older ones fail at cuModuleLoadData (the shim then reports no kernels). + * + * Launch order on one stream, all set up by racer_gpu_decrypt in gpu_cuda.c: + * 1. racer_gcm_partials one thread per GCM_CHUNK of the sealed page + * 2. racer_gcm_finalize one block of 256 threads; writes the verdict + * 3. racer_gcm_ctr one thread per 16-byte block; writes plaintext only + * when the verdict on the device is GCM_VERDICT_PASS + * No host round trip sits between verification and decryption. */ +#include "gcm_core.h" + +__device__ const uint8_t racer_gcm_sbox[256] = GCM_SBOX_INIT; + +#define FINALIZE_THREADS 256 + +extern "C" __global__ void racer_gcm_partials(const uint8_t *sealed, uint64_t n, + const struct gcm_key *key, + struct gcm_partial *partials, uint64_t chunks) { + __shared__ uint64_t table[256]; + for (unsigned i = threadIdx.x; i < 256; i += blockDim.x) table[i] = gcm_crc64_entry(i); + __syncthreads(); + uint64_t j = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (j < chunks) gcm_chunk_partial(table, key, sealed, n, j, &partials[j]); +} + +extern "C" __global__ void racer_gcm_finalize(const uint8_t *sealed, uint64_t n, + const struct gcm_key *key, + const struct gcm_partial *partials, uint64_t chunks, + uint32_t check_crc, uint64_t expected_crc, + struct gcm_verdict *verdict) { + __shared__ uint64_t hi[FINALIZE_THREADS], lo[FINALIZE_THREADS], crc[FINALIZE_THREADS]; + unsigned t = threadIdx.x; + uint64_t h = 0, l = 0, c = 0; + for (uint64_t j = t; j < chunks; j += FINALIZE_THREADS) { + struct gcm_block b = gcm_load(partials[j].ghash); + h ^= b.hi; + l ^= b.lo; + c ^= partials[j].crc; + } + hi[t] = h; + lo[t] = l; + crc[t] = c; + __syncthreads(); + for (unsigned stride = FINALIZE_THREADS / 2; stride; stride >>= 1) { + if (t < stride) { + hi[t] ^= hi[t + stride]; + lo[t] ^= lo[t + stride]; + crc[t] ^= crc[t + stride]; + } + __syncthreads(); + } + if (t == 0) { + uint8_t ghash[16]; + struct gcm_block b = {hi[0], lo[0]}; + gcm_store(ghash, b); + gcm_finalize(key, sealed, n, ghash, crc[0], check_crc, expected_crc, verdict); + } +} + +extern "C" __global__ void racer_gcm_ctr(const uint8_t *sealed, uint64_t n, + const struct gcm_key *key, + const struct gcm_verdict *verdict, uint8_t *out) { + __shared__ uint8_t sbox[256]; + __shared__ struct gcm_key k; + /* Same value for every thread, so the whole block leaves together. */ + if (verdict->status != GCM_VERDICT_PASS) return; + uint8_t *raw = (uint8_t *)&k; + const uint8_t *from = (const uint8_t *)key; + for (unsigned i = threadIdx.x; i < 256; i += blockDim.x) sbox[i] = racer_gcm_sbox[i]; + for (unsigned i = threadIdx.x; i < sizeof(k); i += blockDim.x) raw[i] = from[i]; + __syncthreads(); + uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < (n + 15) / 16) gcm_ctr_block(sbox, &k, sealed, n, i, out); + __syncthreads(); + /* Shared memory outlives the block; do not leave round keys behind. */ + for (unsigned j = threadIdx.x; j < sizeof(k); j += blockDim.x) raw[j] = 0; +} diff --git a/cmd/racer-dataplane/gpu/native/gpu_cuda.c b/cmd/racer-dataplane/gpu/native/gpu_cuda.c new file mode 100644 index 0000000000..fcbe1066a6 --- /dev/null +++ b/cmd/racer-dataplane/gpu/native/gpu_cuda.c @@ -0,0 +1,776 @@ +/* Thin C shim between racer-gpu (Rust) and the CUDA driver API, NVML and + * cuFile. Rust loads it as libracer_gpu_cuda.so.1; see gpu_cuda.h for the + * ABI and result contract. + * + * The shim itself dlopens libcuda.so.1, libnvidia-ml.so.1 and libcufile.so.0 + * with the declarations in cuda_private.h, so it builds without the CUDA + * toolkit. The decrypt kernels are PTX embedded at build time when + * RACER_GPU_PTX names a PTX file (see build-native-cuda.sh); without it, every + * other call works and racer_gpu_decrypt returns RACER_GPU_ENOKERNEL. + * + * Every call runs on the dataplane's native helper thread. Each call pushes + * the owning context and pops it before returning. */ +#define _GNU_SOURCE +#include "gpu_cuda.h" +#include "cuda_private.h" +#include "gcm_core.h" + +#include +#include +#include +#include +#include +#include + +#ifdef RACER_GPU_PTX +/* Embed the PTX image, NUL terminated as cuModuleLoadData requires. */ +__asm__(".section .rodata.racer_gpu_ptx,\"a\"\n" + ".globl racer_gpu_ptx_image\n" + ".hidden racer_gpu_ptx_image\n" + "racer_gpu_ptx_image:\n" + ".incbin \"" RACER_GPU_PTX "\"\n" + ".byte 0\n" + ".previous\n"); +extern const char racer_gpu_ptx_image[]; +#define PTX_IMAGE racer_gpu_ptx_image +#else +#define PTX_IMAGE NULL +#endif + +/* Function pointer types and tables for each library. */ +#define X_TYPE(name, symbol, ret, args) typedef ret(*name##_fn) args; +#define X_FIELD(name, symbol, ret, args) name##_fn name; +#define X_LOAD(name, symbol, ret, args) \ + if (!(table.name = (name##_fn)dlsym(table.lib, symbol))) goto missing; +RACER_CUDA_FUNCTIONS(X_TYPE) +RACER_NVML_FUNCTIONS(X_TYPE) +RACER_CUFILE_FUNCTIONS(X_TYPE) + +struct cuda_table { void *lib; RACER_CUDA_FUNCTIONS(X_FIELD) }; +struct nvml_table { void *lib; RACER_NVML_FUNCTIONS(X_FIELD) }; +struct cufile_table { void *lib; int opened; RACER_CUFILE_FUNCTIONS(X_FIELD) }; + +static struct cuda_table cu; +static struct cufile_table cf; +static pthread_mutex_t load_lock = PTHREAD_MUTEX_INITIALIZER; +static __thread int32_t last_error; + +static const uint8_t shim_sbox[256] = GCM_SBOX_INIT; + +#define KIND_IMPORT 0x494d5054u +#define KIND_STAGING 0x53544147u +#define STAGING_ALIGN (2ull << 20) +/* Workspace layout: key, verdict, then one partial per chunk. */ +#define WORK_KEY 0u +#define WORK_VERDICT 512u +#define WORK_PARTIALS 1024u +#define PARTIAL_THREADS 128u +#define FINALIZE_THREADS 256u +#define CTR_THREADS 256u + +_Static_assert(sizeof(struct gcm_key) <= WORK_VERDICT, "workspace key slot"); +_Static_assert(sizeof(struct gcm_partial) == 32, "partial layout"); +_Static_assert(sizeof(struct gcm_verdict) == sizeof(struct racer_gpu_verdict), "verdict layout"); + +struct gpu_context { + CUcontext ctx; + CUdevice dev; + int32_t ordinal; + int module_state; /* 0 untried, 1 loaded, -1 unavailable */ + CUmodule module; + CUfunction partials, finalize, ctr; +}; + +struct gpu_buffer { + uint32_t kind; + struct gpu_context *context; + CUdeviceptr base; + uint64_t size; +}; + +struct gpu_import { + struct gpu_buffer b; + CUmemGenericAllocationHandle handle; + int has_handle, reserved, mapped; +}; + +struct gpu_staging { + struct gpu_buffer b; + uint64_t allocated; + int registered; +}; + +struct gpu_stream { + struct gpu_context *context; + CUstream stream; + CUdeviceptr work; + uint64_t work_size; + struct racer_gpu_verdict *verdict; /* pinned host memory */ + int decrypt; /* 0 none, 1 queued, 2 synchronized */ +}; + +struct gpu_file { + CUfileHandle_t handle; +}; + +uint32_t racer_gpu_abi(void) { return RACER_GPU_ABI_VERSION; } + +int32_t racer_gpu_last_driver_error(void) { return last_error; } + +/* Map a driver result to a shim code, remembering the raw value. */ +static int drv(CUresult result) { + if (result == CUDA_SUCCESS) return RACER_GPU_OK; + last_error = result; + return RACER_GPU_EDRIVER; +} + +static void wipe(void *p, size_t n) { + volatile uint8_t *v = (volatile uint8_t *)p; + while (n--) *v++ = 0; +} + +int racer_gpu_init(void) { + int rc = RACER_GPU_OK; + pthread_mutex_lock(&load_lock); + if (!cu.lib) { + struct cuda_table table; + memset(&table, 0, sizeof(table)); + table.lib = dlopen("libcuda.so.1", RTLD_NOW | RTLD_LOCAL); + if (!table.lib) { + rc = RACER_GPU_EUNAVAILABLE; + goto out; + } + RACER_CUDA_FUNCTIONS(X_LOAD) + if (drv(table.cuInit(0))) { + dlclose(table.lib); + rc = RACER_GPU_EUNAVAILABLE; + goto out; + } + cu = table; + goto out; + missing: + dlclose(table.lib); + rc = RACER_GPU_EUNAVAILABLE; + } +out: + pthread_mutex_unlock(&load_lock); + return rc; +} + +/* Load NVML for one discovery. Returns 0 when usable. */ +static int nvml_open(struct nvml_table *out) { + struct nvml_table table; + memset(&table, 0, sizeof(table)); + table.lib = dlopen("libnvidia-ml.so.1", RTLD_NOW | RTLD_LOCAL); + if (!table.lib) return -1; + RACER_NVML_FUNCTIONS(X_LOAD) + if (table.nvmlInit() != NVML_SUCCESS) goto missing; + *out = table; + return 0; +missing: + dlclose(table.lib); + return -1; +} + +/* NUMA node from sysfs for a lowercase PCI address, or -1. */ +static int32_t sysfs_numa(const char *pci) { + char path[96]; + int node = -1; + if (snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/numa_node", pci) >= (int)sizeof(path)) + return -1; + FILE *f = fopen(path, "re"); + if (!f) return -1; + if (fscanf(f, "%d", &node) != 1) node = -1; + fclose(f); + return node < 0 ? -1 : node; +} + +static int describe(int ordinal, struct racer_gpu_device *d, struct nvml_table *nvml, int have_nvml) { + CUdevice dev; + CUuuid uuid; + size_t total = 0; + int value = 0; + memset(d, 0, sizeof(*d)); + d->ordinal = ordinal; + d->numa_node = -1; + d->compute_mode = -1; + if (drv(cu.cuDeviceGet(&dev, ordinal)) || drv(cu.cuDeviceGetUuid(&uuid, dev)) || + drv(cu.cuDeviceGetPCIBusId(d->pci_bus_id, sizeof(d->pci_bus_id), dev)) || + drv(cu.cuDeviceTotalMem(&total, dev))) + return -1; + d->pci_bus_id[sizeof(d->pci_bus_id) - 1] = 0; + for (char *p = d->pci_bus_id; *p; ++p) + if (*p >= 'A' && *p <= 'F') *p = (char)(*p - 'A' + 'a'); + memcpy(d->uuid, uuid.bytes, 16); + d->total_memory = total; + static const struct { int attribute; uint32_t flag; } caps[] = { + {CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, RACER_GPU_FLAG_VMM}, + {CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR_SUPPORTED, RACER_GPU_FLAG_POSIX_FD}, + {CU_DEVICE_ATTRIBUTE_DMA_BUF_SUPPORTED, RACER_GPU_FLAG_DMA_BUF}, + {CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_SUPPORTED, RACER_GPU_FLAG_GDR}, + }; + for (size_t i = 0; i < sizeof(caps) / sizeof(caps[0]); ++i) + if (!cu.cuDeviceGetAttribute(&value, caps[i].attribute, dev) && value) d->flags |= caps[i].flag; + unsigned int node; + nvmlDevice_t handle; + if (have_nvml && nvml->nvmlDeviceGetHandleByPciBusId(d->pci_bus_id, &handle) == NVML_SUCCESS && + nvml->nvmlDeviceGetNumaNodeId(handle, &node) == NVML_SUCCESS && node <= INT32_MAX) + d->numa_node = (int32_t)node; + else + d->numa_node = sysfs_numa(d->pci_bus_id); + /* Last, so any earlier failure leaves the device unusable (-1). */ + if (drv(cu.cuDeviceGetAttribute(&value, CU_DEVICE_ATTRIBUTE_COMPUTE_MODE, dev))) return -1; + d->compute_mode = value; + return 0; +} + +int racer_gpu_discover(struct racer_gpu_device *out, uint32_t capacity) { + int count = 0; + struct nvml_table nvml; + if (!cu.lib) return RACER_GPU_EUNAVAILABLE; + if (drv(cu.cuDeviceGetCount(&count))) return RACER_GPU_EDRIVER; + int have_nvml = nvml_open(&nvml) == 0; + for (int i = 0; i < count && (uint32_t)i < capacity; ++i) (void)describe(i, &out[i], &nvml, have_nvml); + if (have_nvml) { + (void)nvml.nvmlShutdown(); + dlclose(nvml.lib); + } + return count; +} + +static int push(struct gpu_context *c) { return drv(cu.cuCtxPushCurrent(c->ctx)); } + +static void pop(void) { + CUcontext previous; + (void)cu.cuCtxPopCurrent(&previous); +} + +int racer_gpu_context(int32_t ordinal, void **out) { + if (!cu.lib) return RACER_GPU_EUNAVAILABLE; + if (!out || ordinal < 0) return RACER_GPU_EINVAL; + *out = NULL; + struct gpu_context *c = calloc(1, sizeof(*c)); + if (!c) return RACER_GPU_ENOMEM; + c->ordinal = ordinal; + int rc = drv(cu.cuDeviceGet(&c->dev, ordinal)); + if (!rc) rc = drv(cu.cuDevicePrimaryCtxRetain(&c->ctx, c->dev)); + if (rc) { + free(c); + return rc; + } + *out = c; + return RACER_GPU_OK; +} + +int racer_gpu_context_free(void *context) { + struct gpu_context *c = context; + if (!c) return RACER_GPU_EINVAL; + if (c->module_state == 1) { + int rc = push(c); + if (rc) return rc; + rc = drv(cu.cuModuleUnload(c->module)); + pop(); + if (rc) return rc; + c->module_state = 0; + } + int rc = drv(cu.cuDevicePrimaryCtxRelease(c->dev)); + if (!rc) free(c); + return rc; +} + +/* Load the decrypt kernels into a context. Context must be current. */ +static int kernels(struct gpu_context *c) { + if (c->module_state == 0) { + const char *image = PTX_IMAGE; + c->module_state = -1; + if (!image) return RACER_GPU_ENOKERNEL; + if (drv(cu.cuModuleLoadData(&c->module, image))) return RACER_GPU_ENOKERNEL; + if (drv(cu.cuModuleGetFunction(&c->partials, c->module, "racer_gcm_partials")) || + drv(cu.cuModuleGetFunction(&c->finalize, c->module, "racer_gcm_finalize")) || + drv(cu.cuModuleGetFunction(&c->ctr, c->module, "racer_gcm_ctr"))) { + /* Keep it as loaded (state 1) if unload fails so free retries. */ + if (drv(cu.cuModuleUnload(c->module))) c->module_state = 1; + return RACER_GPU_ENOKERNEL; + } + c->module_state = 1; + } + return c->module_state == 1 && c->ctr ? RACER_GPU_OK : RACER_GPU_ENOKERNEL; +} + +int racer_gpu_stream(void *context, void **out) { + struct gpu_context *c = context; + if (!c || !out) return RACER_GPU_EINVAL; + *out = NULL; + struct gpu_stream *s = calloc(1, sizeof(*s)); + if (!s) return RACER_GPU_ENOMEM; + s->context = c; + int rc = push(c); + if (rc) { + free(s); + return rc; + } + rc = drv(cu.cuStreamCreate(&s->stream, CU_STREAM_NON_BLOCKING)); + if (!rc) { + rc = drv(cu.cuMemAllocHost((void **)&s->verdict, sizeof(*s->verdict))); + if (rc && drv(cu.cuStreamDestroy(s->stream))) rc = RACER_GPU_ELEAKED; + } + pop(); + if (rc) { + free(s); + return rc; + } + memset(s->verdict, 0, sizeof(*s->verdict)); + *out = s; + return RACER_GPU_OK; +} + +int racer_gpu_stream_sync(void *stream) { + struct gpu_stream *s = stream; + if (!s) return RACER_GPU_EINVAL; + int rc = push(s->context); + if (rc) return rc; + rc = drv(cu.cuStreamSynchronize(s->stream)); + pop(); + if (!rc && s->decrypt == 1) s->decrypt = 2; + return rc; +} + +int racer_gpu_stream_free(void *stream) { + struct gpu_stream *s = stream; + if (!s) return RACER_GPU_EINVAL; + int rc = push(s->context); + if (rc) return rc; + rc = drv(cu.cuStreamSynchronize(s->stream)); + if (!rc && s->work) { + rc = drv(cu.cuMemFree(s->work)); + if (!rc) s->work = 0, s->work_size = 0; + } + if (!rc && s->verdict) { + rc = drv(cu.cuMemFreeHost(s->verdict)); + if (!rc) s->verdict = NULL; + } + if (!rc) rc = drv(cu.cuStreamDestroy(s->stream)); + pop(); + if (!rc) free(s); + return rc; +} + +/* Undo an import as far as possible. Context must be current. */ +static int import_release(struct gpu_import *a) { + if (a->mapped) { + if (drv(cu.cuMemUnmap(a->b.base, a->b.size))) return RACER_GPU_EDRIVER; + a->mapped = 0; + } + if (a->reserved) { + if (drv(cu.cuMemAddressFree(a->b.base, a->b.size))) return RACER_GPU_EDRIVER; + a->reserved = 0; + } + if (a->has_handle) { + if (drv(cu.cuMemRelease(a->handle))) return RACER_GPU_EDRIVER; + a->has_handle = 0; + } + return RACER_GPU_OK; +} + +int racer_gpu_import(void *context, int fd, uint64_t size, void **out) { + struct gpu_context *c = context; + if (!c || fd < 0 || !size || size > SIZE_MAX || !out) return RACER_GPU_EINVAL; + *out = NULL; + struct gpu_import *a = calloc(1, sizeof(*a)); + if (!a) return RACER_GPU_ENOMEM; + a->b.kind = KIND_IMPORT; + a->b.context = c; + a->b.size = size; + int rc = push(c); + if (rc) { + free(a); + return rc; + } + CUmemAllocationProp prop; + size_t granularity = 0; + memset(&prop, 0, sizeof(prop)); + rc = drv(cu.cuMemImportFromShareableHandle(&a->handle, (void *)(uintptr_t)fd, + CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR)); + if (rc) goto fail; + a->has_handle = 1; + rc = drv(cu.cuMemGetAllocationPropertiesFromHandle(&prop, a->handle)); + if (rc) goto fail; + if (prop.location.type != CU_MEM_LOCATION_TYPE_DEVICE || prop.location.id != c->ordinal) { + rc = RACER_GPU_EDEVICE; + goto fail; + } + rc = drv(cu.cuMemGetAllocationGranularity(&granularity, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM)); + if (rc) goto fail; + if (!granularity || size % granularity) { + rc = RACER_GPU_EINVAL; + goto fail; + } + rc = drv(cu.cuMemAddressReserve(&a->b.base, size, 0, 0, 0)); + if (rc) goto fail; + a->reserved = 1; + /* Fails if `size` exceeds the allocation, so we never map past it. */ + rc = drv(cu.cuMemMap(a->b.base, size, 0, a->handle, 0)); + if (rc) goto fail; + a->mapped = 1; + CUmemAccessDesc access; + access.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + access.location.id = c->ordinal; + access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; + rc = drv(cu.cuMemSetAccess(a->b.base, size, &access, 1)); + if (rc) goto fail; + pop(); + *out = a; + return RACER_GPU_OK; +fail: + if (import_release(a)) rc = RACER_GPU_ELEAKED; + else free(a); + pop(); + return rc; +} + +int racer_gpu_import_free(void *allocation) { + struct gpu_import *a = allocation; + if (!a || a->b.kind != KIND_IMPORT) return RACER_GPU_EINVAL; + int rc = push(a->b.context); + if (rc) return rc; + rc = import_release(a); + pop(); + if (!rc) { + a->b.kind = 0; + free(a); + } + return rc; +} + +int racer_gpu_staging(void *context, uint64_t size, void **out) { + struct gpu_context *c = context; + if (!c || !size || size > UINT64_MAX - STAGING_ALIGN || !out) return RACER_GPU_EINVAL; + *out = NULL; + uint64_t allocated = (size + STAGING_ALIGN - 1) & ~(STAGING_ALIGN - 1); + if (allocated > SIZE_MAX) return RACER_GPU_EINVAL; + struct gpu_staging *b = calloc(1, sizeof(*b)); + if (!b) return RACER_GPU_ENOMEM; + b->b.kind = KIND_STAGING; + b->b.context = c; + b->b.size = size; + b->allocated = allocated; + int rc = push(c); + if (rc) { + free(b); + return rc; + } + rc = drv(cu.cuMemAlloc(&b->b.base, allocated)); + /* dma-buf export and cuFile want page alignment; 2 MiB sizes get 2 MiB + * aligned addresses in practice, but check rather than assume. */ + if (!rc && b->b.base % 65536) rc = drv(cu.cuMemFree(b->b.base)) ? RACER_GPU_ELEAKED : RACER_GPU_EINVAL; + pop(); + if (rc) { + if (rc != RACER_GPU_ELEAKED) free(b); + return rc; + } + *out = b; + return RACER_GPU_OK; +} + +int racer_gpu_staging_free(void *staging) { + struct gpu_staging *b = staging; + if (!b || b->b.kind != KIND_STAGING) return RACER_GPU_EINVAL; + if (b->registered) return RACER_GPU_EBUSY; + int rc = push(b->b.context); + if (rc) return rc; + rc = drv(cu.cuMemFree(b->b.base)); + pop(); + if (!rc) { + b->b.kind = 0; + free(b); + } + return rc; +} + +/* Nonempty [offset, offset + length) inside the buffer. */ +static int span(const struct gpu_buffer *b, uint64_t offset, uint64_t length) { + return length && offset <= b->size && length <= b->size - offset ? RACER_GPU_OK : RACER_GPU_ERANGE; +} + +static int buffer(void *p, struct gpu_buffer **out) { + struct gpu_buffer *b = p; + if (!b || (b->kind != KIND_IMPORT && b->kind != KIND_STAGING)) return RACER_GPU_EINVAL; + *out = b; + return RACER_GPU_OK; +} + +int racer_gpu_copy_h2d(void *stream, void *dst, uint64_t offset, const void *src, uint64_t length) { + struct gpu_stream *s = stream; + struct gpu_buffer *d; + if (!s || !src || buffer(dst, &d)) return RACER_GPU_EINVAL; + if (d->context != s->context) return RACER_GPU_EDEVICE; + int rc = span(d, offset, length); + if (rc) return rc; + if ((rc = push(s->context))) return rc; + rc = drv(cu.cuMemcpyHtoDAsync(d->base + offset, src, length, s->stream)); + pop(); + return rc; +} + +int racer_gpu_copy_d2d(void *stream, void *dst, uint64_t dst_offset, void *src, uint64_t src_offset, + uint64_t length) { + struct gpu_stream *s = stream; + struct gpu_buffer *d, *from; + if (!s || buffer(dst, &d) || buffer(src, &from) || from->kind != KIND_STAGING) return RACER_GPU_EINVAL; + if (d->context != s->context || from->context != s->context) return RACER_GPU_EDEVICE; + int rc = span(d, dst_offset, length); + if (!rc) rc = span(from, src_offset, length); + if (rc) return rc; + if ((rc = push(s->context))) return rc; + rc = drv(cu.cuMemcpyDtoDAsync(d->base + dst_offset, from->base + src_offset, length, s->stream)); + pop(); + return rc; +} + +int racer_gpu_copy_d2h(void *stream, void *dst, void *src, uint64_t src_offset, uint64_t length) { + struct gpu_stream *s = stream; + struct gpu_buffer *from; + if (!s || !dst || buffer(src, &from) || from->kind != KIND_STAGING) return RACER_GPU_EINVAL; + if (from->context != s->context) return RACER_GPU_EDEVICE; + int rc = span(from, src_offset, length); + if (rc) return rc; + if ((rc = push(s->context))) return rc; + rc = drv(cu.cuMemcpyDtoHAsync(dst, from->base + src_offset, length, s->stream)); + if (!rc) rc = drv(cu.cuStreamSynchronize(s->stream)); + pop(); + if (!rc && s->decrypt == 1) s->decrypt = 2; + return rc; +} + +static int launch(CUfunction f, uint64_t grid, unsigned threads, CUstream stream, void **args) { + return drv(cu.cuLaunchKernel(f, (unsigned)grid, 1, 1, threads, 1, 1, 0, stream, args, NULL)); +} + +int racer_gpu_decrypt(void *stream, const struct racer_gpu_decrypt *p, void *src, uint64_t src_offset, + void *dst, uint64_t dst_offset) { + struct gpu_stream *s = stream; + struct gpu_buffer *d, *from; + if (!s || !p || buffer(dst, &d) || buffer(src, &from) || from->kind != KIND_STAGING) + return RACER_GPU_EINVAL; + if (p->length > RACER_GPU_MAX_DECRYPT || p->aad_length > RACER_GPU_MAX_AAD || + (!p->aad && p->aad_length) || p->check_crc > 1) + return RACER_GPU_EINVAL; + if (d->context != s->context || from->context != s->context) return RACER_GPU_EDEVICE; + if (s->decrypt == 1) return RACER_GPU_EBUSY; + uint64_t n = p->length; + int rc = span(from, src_offset, n + 16); + if (!rc && n) rc = span(d, dst_offset, n); + if (!rc && !n && dst_offset > d->size) rc = RACER_GPU_ERANGE; + if (rc) return rc; + if ((rc = push(s->context))) return rc; + if ((rc = kernels(s->context))) goto out; + + uint64_t chunks = gcm_chunks(n); + uint64_t need = WORK_PARTIALS + chunks * sizeof(struct gcm_partial); + if (s->work_size < need) { + /* Earlier work may still read the old workspace. */ + if ((rc = drv(cu.cuStreamSynchronize(s->stream)))) goto out; + if (s->work) { + if ((rc = drv(cu.cuMemFree(s->work)))) goto out; + s->work = 0; + s->work_size = 0; + } + if ((rc = drv(cu.cuMemAlloc(&s->work, need)))) goto out; + s->work_size = need; + } + + struct gcm_key key; + gcm_prepare(shim_sbox, p->key, p->nonce, p->aad, p->aad_length, n, &key); + /* Pageable source: the driver has copied `key` when this returns. */ + rc = drv(cu.cuMemcpyHtoDAsync(s->work + WORK_KEY, &key, sizeof(key), s->stream)); + wipe(&key, sizeof(key)); + if (rc) goto out; + s->decrypt = 1; + CUdeviceptr sealed = from->base + src_offset, out_ptr = d->base + dst_offset; + CUdeviceptr key_ptr = s->work + WORK_KEY, verdict = s->work + WORK_VERDICT; + CUdeviceptr partials = s->work + WORK_PARTIALS; + uint32_t check = p->check_crc; + uint64_t expected = p->expected_crc; + void *partial_args[] = {&sealed, &n, &key_ptr, &partials, &chunks}; + void *finalize_args[] = {&sealed, &n, &key_ptr, &partials, &chunks, &check, &expected, &verdict}; + void *ctr_args[] = {&sealed, &n, &key_ptr, &verdict, &out_ptr}; + uint64_t blocks = (n + 15) / 16; + rc = drv(cu.cuMemsetD8Async(verdict, 0, sizeof(struct gcm_verdict), s->stream)); + if (!rc) + rc = launch(s->context->partials, (chunks + PARTIAL_THREADS - 1) / PARTIAL_THREADS, + PARTIAL_THREADS, s->stream, partial_args); + if (!rc) rc = launch(s->context->finalize, 1, FINALIZE_THREADS, s->stream, finalize_args); + if (!rc && blocks) + rc = launch(s->context->ctr, (blocks + CTR_THREADS - 1) / CTR_THREADS, CTR_THREADS, s->stream, + ctr_args); + /* Always queue the wipe of key material and partials, even on failure. */ + int wiped = drv(cu.cuMemsetD8Async(key_ptr, 0, sizeof(struct gcm_key), s->stream)); + if (!wiped) + wiped = drv(cu.cuMemsetD8Async(partials, 0, chunks * sizeof(struct gcm_partial), s->stream)); + if (!rc) rc = wiped; + if (!rc) + rc = drv(cu.cuMemcpyDtoHAsync(s->verdict, verdict, sizeof(*s->verdict), s->stream)); +out: + pop(); + return rc; +} + +int racer_gpu_verdict(void *stream, struct racer_gpu_verdict *out) { + struct gpu_stream *s = stream; + if (!s || !out) return RACER_GPU_EINVAL; + if (s->decrypt != 2) return RACER_GPU_EBUSY; + *out = *s->verdict; + memset(s->verdict, 0, sizeof(*s->verdict)); + s->decrypt = 0; + return RACER_GPU_OK; +} + +static int file_error(CUfileError_t e) { + if (e.err == CU_FILE_SUCCESS) return RACER_GPU_OK; + last_error = e.err; + return RACER_GPU_EDRIVER; +} + +/* Set once cuFile fails to load or open. Opening the driver probes every GPU + * for P2PDMA, which can take most of a second, so a failure is not retried. */ +static int cf_failed; + +int racer_gpu_file_open(void) { + int rc = RACER_GPU_OK; + pthread_mutex_lock(&load_lock); + if (cf_failed) { + rc = RACER_GPU_EUNAVAILABLE; + } else if (!cf.opened) { + struct cufile_table table; + memset(&table, 0, sizeof(table)); + table.lib = dlopen("libcufile.so.0", RTLD_NOW | RTLD_LOCAL); + if (!table.lib) { + cf_failed = 1; + rc = RACER_GPU_EUNAVAILABLE; + goto out; + } + RACER_CUFILE_FUNCTIONS(X_LOAD) + /* Without nvidia-fs or P2PDMA, cuFile's compatibility mode does a + * POSIX read into a bounce buffer and a copy on this helper thread, + * so every GPU job waits behind the disk. Racer's own host path does + * that work on I/O threads. Turn it off unless the operator chose a + * value with CUFILE_ALLOW_COMPAT_MODE; cuFileDriverOpen then fails + * without GPUDirect Storage and Racer uses its host path. */ + cuFileSetParameterBool_fn set_bool = + (cuFileSetParameterBool_fn)dlsym(table.lib, "cuFileSetParameterBool"); + if (set_bool && !getenv("CUFILE_ALLOW_COMPAT_MODE")) + set_bool(CUFILE_PARAM_PROPERTIES_ALLOW_COMPAT_MODE, false); + if (file_error(table.cuFileDriverOpen())) { + dlclose(table.lib); + cf_failed = 1; + rc = RACER_GPU_EUNAVAILABLE; + goto out; + } + /* Left open for the life of the process. */ + table.opened = 1; + cf = table; + goto out; + missing: + dlclose(table.lib); + cf_failed = 1; + rc = RACER_GPU_EUNAVAILABLE; + } +out: + pthread_mutex_unlock(&load_lock); + return rc; +} + +int racer_gpu_file(int fd, void **out) { + if (!cf.opened) return RACER_GPU_EUNAVAILABLE; + if (fd < 0 || !out) return RACER_GPU_EINVAL; + *out = NULL; + struct gpu_file *f = calloc(1, sizeof(*f)); + if (!f) return RACER_GPU_ENOMEM; + CUfileDescr_t descr; + memset(&descr, 0, sizeof(descr)); + descr.type = CU_FILE_HANDLE_TYPE_OPAQUE_FD; + descr.handle.fd = fd; + int rc = file_error(cf.cuFileHandleRegister(&f->handle, &descr)); + if (rc) { + free(f); + return rc; + } + *out = f; + return RACER_GPU_OK; +} + +int racer_gpu_file_free(void *file) { + struct gpu_file *f = file; + if (!f || !cf.opened) return RACER_GPU_EINVAL; + cf.cuFileHandleDeregister(f->handle); + free(f); + return RACER_GPU_OK; +} + +static int staging(void *p, struct gpu_staging **out) { + struct gpu_staging *b = p; + if (!b || b->b.kind != KIND_STAGING) return RACER_GPU_EINVAL; + *out = b; + return RACER_GPU_OK; +} + +int racer_gpu_staging_register(void *p) { + struct gpu_staging *b; + if (!cf.opened) return RACER_GPU_EUNAVAILABLE; + if (staging(p, &b)) return RACER_GPU_EINVAL; + if (b->registered) return RACER_GPU_EBUSY; + int rc = push(b->b.context); + if (rc) return rc; + rc = file_error(cf.cuFileBufRegister((const void *)(uintptr_t)b->b.base, b->allocated, 0)); + pop(); + if (!rc) b->registered = 1; + return rc; +} + +int racer_gpu_staging_unregister(void *p) { + struct gpu_staging *b; + if (!cf.opened || staging(p, &b)) return RACER_GPU_EINVAL; + if (!b->registered) return RACER_GPU_EBUSY; + int rc = push(b->b.context); + if (rc) return rc; + rc = file_error(cf.cuFileBufDeregister((const void *)(uintptr_t)b->b.base)); + pop(); + if (!rc) b->registered = 0; + return rc; +} + +int64_t racer_gpu_file_read(void *file, void *p, uint64_t offset, uint64_t file_offset, uint64_t length) { + struct gpu_file *f = file; + struct gpu_staging *b; + if (!cf.opened) return RACER_GPU_EUNAVAILABLE; + if (!f || staging(p, &b) || file_offset > (uint64_t)INT64_MAX - length) return RACER_GPU_EINVAL; + int rc = span(&b->b, offset, length); + if (rc) return rc; + if ((rc = push(b->b.context))) return rc; + ssize_t n = cf.cuFileRead(f->handle, (void *)(uintptr_t)b->b.base, length, (off_t)file_offset, + (off_t)offset); + pop(); + if (n < 0) { + last_error = (int32_t)n; + return RACER_GPU_EIO; + } + return (uint64_t)n == length ? (int64_t)n : RACER_GPU_EIO; +} + +int racer_gpu_dmabuf(void *p, uint64_t offset, uint64_t length, int *fd) { + struct gpu_staging *b; + long page = sysconf(_SC_PAGESIZE); + if (staging(p, &b) || !fd || page <= 0) return RACER_GPU_EINVAL; + if (!length || offset % (uint64_t)page || length % (uint64_t)page || offset > b->allocated || + length > b->allocated - offset) + return RACER_GPU_ERANGE; + *fd = -1; + int rc = push(b->b.context); + if (rc) return rc; + int handle = -1; + rc = drv(cu.cuMemGetHandleForAddressRange(&handle, b->b.base + offset, length, + CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, 0)); + pop(); + if (!rc) *fd = handle; + return rc; +} diff --git a/cmd/racer-dataplane/gpu/native/gpu_cuda.h b/cmd/racer-dataplane/gpu/native/gpu_cuda.h new file mode 100644 index 0000000000..220d6a1bb0 --- /dev/null +++ b/cmd/racer-dataplane/gpu/native/gpu_cuda.h @@ -0,0 +1,156 @@ +/* C ABI of libracer_gpu_cuda.so.1, loaded by racer-gpu (src/ffi.rs) with + * dlopen. The simulation backend (src/simulation.rs) implements the same + * functions in Rust. + * + * Bump RACER_GPU_ABI_VERSION whenever a struct, signature, or result + * contract changes. Rust refuses any other version. + * + * Every int result is RACER_GPU_OK or one of the negative codes below. A + * failed free leaves the resource in place and returns a code; Rust then + * leaks the resource on purpose because the GPU may still use it. Every call + * that takes an offset and length checks it against the buffer size, so the + * shim never touches memory outside a buffer. */ +#ifndef RACER_GPU_CUDA_H +#define RACER_GPU_CUDA_H + +#include +#include + +#define RACER_GPU_ABI_VERSION 1u + +#define RACER_GPU_OK 0 +#define RACER_GPU_EINVAL (-1) /* bad argument or handle type */ +#define RACER_GPU_ERANGE (-2) /* offset or length outside a buffer */ +#define RACER_GPU_EUNAVAILABLE (-3) /* driver library missing or init failed */ +#define RACER_GPU_EDEVICE (-4) /* allocation or buffer on another device */ +#define RACER_GPU_EDRIVER (-5) /* CUDA, NVML or cuFile call failed */ +#define RACER_GPU_ENOMEM (-6) /* host or device allocation failed */ +#define RACER_GPU_ENOKERNEL (-7) /* shim built without decrypt kernels */ +#define RACER_GPU_EIO (-8) /* cuFile read failed or was short */ +#define RACER_GPU_EBUSY (-9) /* wrong state, for example no verdict to read */ +#define RACER_GPU_ELEAKED (-10) /* failed, and cleanup leaked an internal resource */ + +/* Device capability bits in racer_gpu_device.flags. */ +#define RACER_GPU_FLAG_VMM 1u /* virtual memory management */ +#define RACER_GPU_FLAG_POSIX_FD 2u /* POSIX fd shareable handles */ +#define RACER_GPU_FLAG_DMA_BUF 4u /* dma-buf export */ +#define RACER_GPU_FLAG_GDR 8u /* GPUDirect RDMA */ + +/* CUcomputemode values (cuda.h). */ +#define RACER_GPU_COMPUTE_DEFAULT 0 +#define RACER_GPU_COMPUTE_PROHIBITED 2 +#define RACER_GPU_COMPUTE_EXCLUSIVE_PROCESS 3 + +/* One CUDA device. Mirrors `RawDevice` in ffi.rs. */ +struct racer_gpu_device { + uint8_t uuid[16]; + char pci_bus_id[32]; /* NUL terminated, for example "0000:3b:00.0" */ + int32_t ordinal; + int32_t numa_node; /* -1 when unknown */ + int32_t compute_mode; + uint32_t flags; + uint64_t total_memory; +}; + +/* One page to verify and decrypt. Mirrors `RawDecrypt` in ffi.rs. + * `key` and `nonce` are the XAES-derived AES-256-GCM key and nonce. The + * source holds `length` ciphertext bytes then the 16-byte tag. */ +struct racer_gpu_decrypt { + uint8_t key[32]; + uint8_t nonce[12]; + uint32_t check_crc; /* 1 to compare CRC-64/XZ with expected_crc */ + uint64_t expected_crc; + uint64_t length; + const uint8_t *aad; + uint64_t aad_length; +}; + +/* Mirrors `RawVerdict` in gcm.rs and `struct gcm_verdict`. */ +struct racer_gpu_verdict { + uint32_t status; + uint32_t reserved; + uint64_t crc; +}; + +_Static_assert(sizeof(struct racer_gpu_device) == 72, "device ABI size"); +_Static_assert(offsetof(struct racer_gpu_device, ordinal) == 48, "device ABI offset"); +_Static_assert(offsetof(struct racer_gpu_device, total_memory) == 64, "device ABI offset"); +_Static_assert(sizeof(struct racer_gpu_decrypt) == 80, "decrypt ABI size"); +_Static_assert(offsetof(struct racer_gpu_decrypt, check_crc) == 44, "decrypt ABI offset"); +_Static_assert(offsetof(struct racer_gpu_decrypt, aad) == 64, "decrypt ABI offset"); +_Static_assert(sizeof(struct racer_gpu_verdict) == 16, "verdict ABI size"); + +/* Largest page one decrypt call accepts (1 GiB). */ +#define RACER_GPU_MAX_DECRYPT (1ull << 30) +/* Largest AAD one decrypt call accepts. */ +#define RACER_GPU_MAX_AAD 4096u + +uint32_t racer_gpu_abi(void); +/* CUresult, NVML or cuFile status of this thread's last failed driver call. */ +int32_t racer_gpu_last_driver_error(void); +/* Load libcuda.so.1 and call cuInit. Safe to call more than once. */ +int racer_gpu_init(void); +/* Fill up to `capacity` devices. Returns the total count, which may exceed + * `capacity`, or a negative code. Reports every device, including ones in + * EXCLUSIVE_PROCESS or PROHIBITED mode; Rust decides which are usable. */ +int racer_gpu_discover(struct racer_gpu_device *out, uint32_t capacity); + +/* Retain the device's primary context. */ +int racer_gpu_context(int32_t ordinal, void **out); +int racer_gpu_context_free(void *context); + +int racer_gpu_stream(void *context, void **out); +int racer_gpu_stream_sync(void *stream); +/* Synchronizes, then frees the stream and its decrypt workspace. */ +int racer_gpu_stream_free(void *stream); + +/* Import a POSIX fd shareable handle of `size` bytes, check it lives on the + * context's device, map all of it and grant this context read-write access. + * The fd stays owned by the caller. */ +int racer_gpu_import(void *context, int fd, uint64_t size, void **out); +/* Unmap, free the address range and release the handle. The caller must + * have synchronized every stream that used it. */ +int racer_gpu_import_free(void *allocation); + +/* Allocate device staging memory, page aligned, usable with cuFile and + * dma-buf export. `size` is rounded up to 2 MiB. */ +int racer_gpu_staging(void *context, uint64_t size, void **out); +int racer_gpu_staging_free(void *staging); + +/* Queue a copy of host bytes into an import or staging buffer. The copy is + * asynchronous: for page-locked memory CUDA may still read the host buffer + * after this returns, so the caller must keep it alive and unchanged until + * the stream is synchronized. The Rust wrapper owns the source until then. */ +int racer_gpu_copy_h2d(void *stream, void *dst, uint64_t offset, const void *src, uint64_t length); +/* Queue a device copy. `src` must be a staging buffer. */ +int racer_gpu_copy_d2d(void *stream, void *dst, uint64_t dst_offset, void *src, uint64_t src_offset, + uint64_t length); +/* Copy staging bytes to the host and wait. Imports are never read. */ +int racer_gpu_copy_d2h(void *stream, void *dst, void *src, uint64_t src_offset, uint64_t length); + +/* Queue verify-then-decrypt of the sealed page at `src + src_offset` (a + * staging buffer) into `dst + dst_offset`. The destination is written only if + * the CRC (when requested) and the tag match; the decision is made on the + * device. One decrypt per stream may be outstanding; read its verdict with + * racer_gpu_verdict after racer_gpu_stream_sync. */ +int racer_gpu_decrypt(void *stream, const struct racer_gpu_decrypt *params, void *src, + uint64_t src_offset, void *dst, uint64_t dst_offset); +int racer_gpu_verdict(void *stream, struct racer_gpu_verdict *out); + +/* Open the cuFile driver (libcufile.so.0). Safe to call more than once. */ +int racer_gpu_file_open(void); +/* Register an O_DIRECT fd with cuFile. The fd stays owned by the caller. */ +int racer_gpu_file(int fd, void **out); +int racer_gpu_file_free(void *file); +/* Register or deregister a staging buffer with cuFile. */ +int racer_gpu_staging_register(void *staging); +int racer_gpu_staging_unregister(void *staging); +/* Read `length` bytes at `file_offset` into staging at `offset`. Returns the + * byte count (always `length` on success) or a negative code. */ +int64_t racer_gpu_file_read(void *file, void *staging, uint64_t offset, uint64_t file_offset, + uint64_t length); +/* Export staging bytes as a dma-buf fd. Offset and length must be multiples + * of the host page size. */ +int racer_gpu_dmabuf(void *staging, uint64_t offset, uint64_t length, int *fd); + +#endif diff --git a/cmd/racer-dataplane/gpu/native/kernel_emulation.cc b/cmd/racer-dataplane/gpu/native/kernel_emulation.cc new file mode 100644 index 0000000000..2a08d42e36 --- /dev/null +++ b/cmd/racer-dataplane/gpu/native/kernel_emulation.cc @@ -0,0 +1,142 @@ +// Runs the CUDA decrypt kernels (gcm_kernel.cu) on the CPU. +// +// Hosted CI has no GPU and no nvcc. This harness compiles the kernel source as +// C++ with one std::thread per CUDA thread, one block at a time, so +// __shared__ data is per block and __syncthreads is a real barrier. It uses +// the launch geometry from gpu_cuda.c and checks the kernels against the +// serial gcm_core.h path, which the Rust tests check against racer-crypto. +// +// Build and run: run-kernel-emulation.sh OUTPUT_DIR +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define __CUDACC__ 1 +#define __host__ +#define __device__ +#define __global__ +#define __shared__ static +struct emu_dim3 { + unsigned x, y, z; +}; +static thread_local emu_dim3 threadIdx; +static thread_local emu_dim3 blockIdx; +static thread_local emu_dim3 blockDim; +static std::barrier<> *emu_barrier; +static void __syncthreads() { emu_barrier->arrive_and_wait(); } + +#include "gcm_kernel.cu" + +static const uint8_t sbox[256] = GCM_SBOX_INIT; +static int failures; + +#define CHECK(cond, ...) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: ", __FILE__, __LINE__); \ + std::fprintf(stderr, __VA_ARGS__); \ + std::fprintf(stderr, "\n"); \ + ++failures; \ + } \ + } while (0) + +// Emulate <<>>. +static void launch(uint64_t grid, unsigned threads, const std::function &kernel) { + for (uint64_t b = 0; b < grid; ++b) { + auto barrier = std::make_unique>(threads); + emu_barrier = barrier.get(); + std::vector pool; + for (unsigned t = 0; t < threads; ++t) + pool.emplace_back([&, t] { + threadIdx = {t, 0, 0}; + blockIdx = {(unsigned)b, 0, 0}; + blockDim = {threads, 1, 1}; + kernel(); + }); + for (auto &thread : pool) thread.join(); + emu_barrier = nullptr; + } +} + +// The gpu_cuda.c sequence: partials, finalize, ctr. +static gcm_verdict run(const gcm_key &key, const std::vector &sealed, uint64_t n, + uint32_t check, uint64_t expected, std::vector &out) { + uint64_t chunks = gcm_chunks(n); + std::vector partials(chunks); + gcm_verdict verdict = {}; + launch((chunks + 127) / 128, 128, + [&] { racer_gcm_partials(sealed.data(), n, &key, partials.data(), chunks); }); + launch(1, 256, [&] { + racer_gcm_finalize(sealed.data(), n, &key, partials.data(), chunks, check, expected, &verdict); + }); + uint64_t blocks = (n + 15) / 16; + if (blocks) + launch((blocks + 255) / 256, 256, + [&] { racer_gcm_ctr(sealed.data(), n, &key, &verdict, out.data()); }); + return verdict; +} + +int main() { + uint64_t state = 0x243f6a8885a308d3ull; + auto next = [&] { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + return (uint8_t)state; + }; + uint64_t table[256]; + gcm_crc64_table(table); + for (uint64_t n : {0ull, 1ull, 15ull, 16ull, 17ull, 4079ull, 4080ull, 4081ull, 8192ull, + 20000ull, 70001ull}) { + uint8_t raw_key[32], nonce[12], aad[77]; + for (auto &b : raw_key) b = next(); + for (auto &b : nonce) b = next(); + for (auto &b : aad) b = next(); + std::vector plaintext(n), sealed(n + 16); + for (auto &b : plaintext) b = next(); + gcm_key key; + gcm_prepare(sbox, raw_key, nonce, aad, sizeof(aad), n, &key); + + // Seal: CTR is its own inverse, then the tag from the serial core. + for (uint64_t i = 0; i < (n + 15) / 16; ++i) gcm_ctr_block(sbox, &key, plaintext.data(), n, i, sealed.data()); + gcm_block s = gcm_load(key.prefix); + for (uint64_t j = 0; j < gcm_chunks(n); ++j) { + gcm_partial p; + gcm_chunk_partial(table, &key, sealed.data(), n, j, &p); + gcm_block g = gcm_load(p.ghash); + s.hi ^= g.hi; + s.lo ^= g.lo; + } + uint8_t tag[16]; + gcm_store(tag, s); + for (int i = 0; i < 16; ++i) sealed[n + i] = tag[i] ^ key.ekj0[i]; + uint64_t crc = gcm_crc64(table, sealed.data(), n + 16); + + std::vector out(n, 0xa5); + gcm_verdict v = run(key, sealed, n, 1, crc, out); + CHECK(v.status == GCM_VERDICT_PASS && v.crc == crc, "n=%llu pass status=%u", (unsigned long long)n, v.status); + CHECK(out == plaintext, "n=%llu plaintext", (unsigned long long)n); + + std::vector untouched(n, 0xa5); + out = untouched; + v = run(key, sealed, n, 1, crc ^ 1, out); + CHECK(v.status == GCM_VERDICT_CRC && out == untouched, "n=%llu crc", (unsigned long long)n); + + auto tampered = sealed; + tampered[n / 2] ^= 4; + v = run(key, tampered, n, 0, 0, out); + CHECK(v.status == GCM_VERDICT_TAG && out == untouched, "n=%llu tag", (unsigned long long)n); + } + if (failures) { + std::fprintf(stderr, "%d kernel emulation failures\n", failures); + return 1; + } + std::printf("kernel emulation passed\n"); + return 0; +} diff --git a/cmd/racer-dataplane/gpu/native/run-kernel-emulation.sh b/cmd/racer-dataplane/gpu/native/run-kernel-emulation.sh new file mode 100755 index 0000000000..91d6abafb7 --- /dev/null +++ b/cmd/racer-dataplane/gpu/native/run-kernel-emulation.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +# +# Compile gcm_kernel.cu as host C++ (kernel_emulation.cc) and run it. +set -eu + +if [ "$#" -ne 1 ]; then + echo "usage: run-kernel-emulation.sh OUTPUT_DIR" >&2 + exit 2 +fi + +native=$(dirname "$0") +mkdir -p "$1" +"${CXX:-c++}" -std=c++20 -O2 -Wall -Wextra -Werror -pthread -I "$native" \ + -o "$1/kernel-emulation" "$native/kernel_emulation.cc" +"$1/kernel-emulation" diff --git a/cmd/racer-dataplane/gpu/src/ffi.rs b/cmd/racer-dataplane/gpu/src/ffi.rs new file mode 100644 index 0000000000..901ec043af --- /dev/null +++ b/cmd/racer-dataplane/gpu/src/ffi.rs @@ -0,0 +1,353 @@ +//! The C ABI of `libracer_gpu_cuda.so.1` (`native/gpu_cuda.h`) and its loader. +//! +//! Every safe type in `lib.rs` holds an `Rc`, which holds the [`Api`] +//! symbol table, so the library stays loaded while any GPU object lives. The +//! simulation backend builds the same table from Rust functions. +use crate::{Error, Result}; + +use std::{ + cell::Cell, + ffi::{c_int, c_void}, + rc::Rc, +}; + +pub(crate) use crate::gcm::RawVerdict; + +/// `RACER_GPU_ABI_VERSION`. +#[cfg(all(feature = "native", target_os = "linux"))] +pub(crate) const ABI_VERSION: u32 = 1; + +/// Result codes from `gpu_cuda.h`. +pub(crate) mod code { + use std::ffi::c_int; + + pub const OK: c_int = 0; + pub const EINVAL: c_int = -1; + pub const ERANGE: c_int = -2; + pub const EUNAVAILABLE: c_int = -3; + pub const EDEVICE: c_int = -4; + pub const EDRIVER: c_int = -5; + pub const ENOMEM: c_int = -6; + pub const ENOKERNEL: c_int = -7; + pub const EIO: c_int = -8; + pub const EBUSY: c_int = -9; + pub const ELEAKED: c_int = -10; +} + +/// Device capability bits (`RACER_GPU_FLAG_*`). +pub mod flag { + /// Virtual memory management (import and map). + pub const VMM: u32 = 1; + /// POSIX fd shareable handles. + pub const POSIX_FD: u32 = 2; + /// dma-buf export. + pub const DMA_BUF: u32 = 4; + /// GPUDirect RDMA. + pub const GDR: u32 = 8; +} + +/// One device. Matches `struct racer_gpu_device`. +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct RawDevice { + pub uuid: [u8; 16], + + pub pci_bus_id: [u8; 32], + + pub ordinal: i32, + + pub numa_node: i32, + + pub compute_mode: i32, + + pub flags: u32, + + pub total_memory: u64, +} + +impl Default for RawDevice { + /// Unknown NUMA node and compute mode. + fn default() -> Self { + Self { + uuid: [0; 16], + pci_bus_id: [0; 32], + ordinal: 0, + numa_node: -1, + compute_mode: -1, + flags: 0, + total_memory: 0, + } + } +} + +/// One decrypt request. Matches `struct racer_gpu_decrypt`. Holds a key, so +/// callers wipe it after the call. +#[repr(C)] +pub(crate) struct RawDecrypt { + pub key: [u8; 32], + + pub nonce: [u8; 12], + + pub check_crc: u32, + + pub expected_crc: u64, + + pub length: u64, + + pub aad: *const u8, + + pub aad_length: u64, +} + +impl Drop for RawDecrypt { + /// Wipe the key and nonce. + fn drop(&mut self) { + use zeroize::Zeroize; + self.key.zeroize(); + self.nonce.zeroize(); + } +} + +/// `RACER_GPU_MAX_DECRYPT`. +pub const MAX_DECRYPT: u64 = 1 << 30; + +/// `RACER_GPU_MAX_AAD`. +pub const MAX_AAD: usize = 4096; + +/// Opaque handle out-parameter. +pub(crate) type Out = *mut *mut c_void; + +/// Build [`Api`] and its symbol loader from `racer_gpu_` functions. +macro_rules! native_api { + ($($field:ident: $signature:ty),* $(,)?) => { + /// The shim's functions. + pub(crate) struct Api { + /// `dlopen` handle; `None` for the simulation. + #[cfg_attr(not(all(feature = "native", target_os = "linux")), allow(dead_code))] + pub(crate) library: Option>, + + $( + pub(crate) $field: $signature, + + )* + } + + impl Api { + #[cfg(all(feature = "native", target_os = "linux"))] + /// Look up every function in `library`. + unsafe fn symbols(library: std::ptr::NonNull) -> Result { + Ok(Self { + library: Some(library), + $($field: { + let pointer = unsafe { libc::dlsym(library.as_ptr(), + concat!("racer_gpu_", stringify!($field), "\0").as_ptr().cast()) }; + if pointer.is_null() { return Err(Error::Unavailable); } + unsafe { std::mem::transmute::<*mut c_void, $signature>(pointer) } + },)* + }) + } + } + }; +} + +native_api! { + last_driver_error: unsafe extern "C" fn() -> i32, + + init: unsafe extern "C" fn() -> c_int, + + discover: unsafe extern "C" fn(*mut RawDevice, u32) -> c_int, + + context: unsafe extern "C" fn(i32, Out) -> c_int, + + context_free: unsafe extern "C" fn(*mut c_void) -> c_int, + + stream: unsafe extern "C" fn(*mut c_void, Out) -> c_int, + + stream_sync: unsafe extern "C" fn(*mut c_void) -> c_int, + + stream_free: unsafe extern "C" fn(*mut c_void) -> c_int, + + import: unsafe extern "C" fn(*mut c_void, c_int, u64, Out) -> c_int, + + import_free: unsafe extern "C" fn(*mut c_void) -> c_int, + + staging: unsafe extern "C" fn(*mut c_void, u64, Out) -> c_int, + + staging_free: unsafe extern "C" fn(*mut c_void) -> c_int, + + copy_h2d: unsafe extern "C" fn(*mut c_void, *mut c_void, u64, *const c_void, u64) -> c_int, + + copy_d2d: unsafe extern "C" fn(*mut c_void, *mut c_void, u64, *mut c_void, u64, u64) -> c_int, + + copy_d2h: unsafe extern "C" fn(*mut c_void, *mut c_void, *mut c_void, u64, u64) -> c_int, + + decrypt: unsafe extern "C" fn( + *mut c_void, + *const RawDecrypt, + *mut c_void, + u64, + *mut c_void, + u64, + ) -> c_int, + + verdict: unsafe extern "C" fn(*mut c_void, *mut RawVerdict) -> c_int, + + file_open: unsafe extern "C" fn() -> c_int, + + file: unsafe extern "C" fn(c_int, Out) -> c_int, + + file_free: unsafe extern "C" fn(*mut c_void) -> c_int, + + staging_register: unsafe extern "C" fn(*mut c_void) -> c_int, + + staging_unregister: unsafe extern "C" fn(*mut c_void) -> c_int, + + file_read: unsafe extern "C" fn(*mut c_void, *mut c_void, u64, u64, u64) -> i64, + + dmabuf: unsafe extern "C" fn(*mut c_void, u64, u64, *mut c_int) -> c_int, +} + +impl Api { + #[cfg(not(all(feature = "native", target_os = "linux")))] + /// No native backend in this build. + pub(crate) fn load() -> Result> { + Err(Error::Unavailable) + } + + #[cfg(all(feature = "native", target_os = "linux"))] + /// Load `libracer_gpu_cuda.so.1` and check its ABI version. Does not + /// initialize CUDA. + pub(crate) fn load() -> Result> { + use std::ptr::NonNull; + + unsafe { + let library = NonNull::new(libc::dlopen( + c"libracer_gpu_cuda.so.1".as_ptr(), + libc::RTLD_NOW | libc::RTLD_LOCAL, + )) + .ok_or(Error::Unavailable)?; + + /// Unloads the library if we bail out early. + struct Guard(NonNull); + + impl Drop for Guard { + /// Unload. + fn drop(&mut self) { + unsafe { + libc::dlclose(self.0.as_ptr()); + } + } + } + + let guard = Guard(library); + let version = libc::dlsym(library.as_ptr(), c"racer_gpu_abi".as_ptr()); + if version.is_null() { + return Err(Error::Unavailable); + } + let version = + std::mem::transmute::<*mut c_void, unsafe extern "C" fn() -> u32>(version); + if version() != ABI_VERSION { + return Err(Error::Unavailable); + } + let api = Self::symbols(library)?; + std::mem::forget(guard); + Ok(Rc::new(api)) + } + } +} + +impl Drop for Api { + /// Unload the library. Leaked objects keep it loaded forever. + fn drop(&mut self) { + #[cfg(all(feature = "native", target_os = "linux"))] + if let Some(library) = self.library { + unsafe { + libc::dlclose(library.as_ptr()); + } + } + } +} + +/// Map a shim result code to an error. `last_driver_error` must be the +/// calling thread's view of the shim, since the driver error is per thread. +pub(crate) fn check(rc: c_int, last_driver_error: unsafe extern "C" fn() -> i32) -> Result<()> { + match rc { + code::OK => Ok(()), + code::EINVAL => Err(Error::InvalidRequest), + code::ERANGE => Err(Error::InvalidRange), + code::EUNAVAILABLE => Err(Error::Unavailable), + code::EDEVICE => Err(Error::WrongDevice), + code::EDRIVER => Err(Error::Driver(unsafe { last_driver_error() })), + code::ENOMEM => Err(Error::OutOfMemory), + code::ENOKERNEL => Err(Error::KernelUnavailable), + code::EIO => Err(Error::Io), + code::EBUSY => Err(Error::Busy), + code::ELEAKED => Err(Error::Leaked), + _ => Err(Error::Driver(rc)), + } +} + +/// The loaded backend and the leak counter every object shares. +pub(crate) struct Shared { + pub(crate) api: Rc, + + pub(crate) leaks: Cell, +} + +impl Shared { + /// Map a shim result code to an error. + pub(crate) fn check(&self, rc: c_int) -> Result<()> { + if rc == code::ELEAKED { + self.leaks.set(self.leaks.get() + 1); + } + check(rc, self.api.last_driver_error) + } + + /// Count a leaked resource and keep the library loaded for good. + pub(crate) fn leak(self: &Rc) { + self.leaks.set(self.leaks.get() + 1); + std::mem::forget(self.clone()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + /// Rust ABI records match the shim's `_Static_assert`s in gpu_cuda.h. + fn abi_layout_matches_c_static_assertions() { + assert_eq!(std::mem::size_of::(), 72); + assert_eq!(std::mem::offset_of!(RawDevice, ordinal), 48); + assert_eq!(std::mem::offset_of!(RawDevice, total_memory), 64); + assert_eq!(std::mem::size_of::(), 80); + assert_eq!(std::mem::offset_of!(RawDecrypt, check_crc), 44); + assert_eq!(std::mem::offset_of!(RawDecrypt, aad), 64); + assert_eq!(std::mem::size_of::(), 16); + } + + #[cfg(all(feature = "native", target_os = "linux"))] + #[test] + #[ignore = "requires libracer_gpu_cuda.so.1 on the loader path"] + /// The installed shim loads, speaks this ABI, and exports every symbol. + /// Without a CUDA driver, every entry point fails cleanly. + fn required_native_shim_loads_abi_and_symbols() { + let api = Api::load().expect("libracer_gpu_cuda.so.1 must be installed for this test"); + assert!(api.library.is_some()); + let rc = unsafe { (api.init)() }; + assert!( + rc == code::OK || rc == code::EUNAVAILABLE, + "init returned {rc}" + ); + if rc == code::EUNAVAILABLE { + let mut device = RawDevice::default(); + assert_eq!( + unsafe { (api.discover)(&mut device, 1) }, + code::EUNAVAILABLE + ); + let mut out = std::ptr::null_mut(); + assert_eq!(unsafe { (api.context)(0, &mut out) }, code::EUNAVAILABLE); + assert_eq!(unsafe { (api.file)(3, &mut out) }, code::EUNAVAILABLE); + } + } +} diff --git a/cmd/racer-dataplane/gpu/src/gcm.rs b/cmd/racer-dataplane/gpu/src/gcm.rs new file mode 100644 index 0000000000..35d429a610 --- /dev/null +++ b/cmd/racer-dataplane/gpu/src/gcm.rs @@ -0,0 +1,154 @@ +//! Host emulation of the GPU decrypt kernels, from `native/gcm_host.c`. +//! +//! It runs the exact `native/gcm_core.h` steps the CUDA kernels run, serially. +//! The simulation backend uses it as its kernel, and the tests compare it with +//! `racer-crypto`. + +use std::ffi::c_int; + +/// Verification result. Matches `struct gcm_verdict` and +/// `struct racer_gpu_verdict` in C. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct RawVerdict { + /// One of the `VERDICT_*` values. + pub status: u32, + + /// Always zero. + pub reserved: u32, + + /// CRC-64/XZ of the sealed page, computed by the kernel. + pub crc: u64, +} + +/// Verification did not run. +pub const VERDICT_NONE: u32 = 0; + +/// CRC (if requested) and tag matched; the plaintext was written. +pub const VERDICT_PASS: u32 = 0x600d; + +/// The CRC did not match. Nothing was written. +pub const VERDICT_CRC: u32 = 2; + +/// The tag did not match. Nothing was written. +pub const VERDICT_TAG: u32 = 3; + +/// Bytes covered by each parallel partial (`GCM_CHUNK`). +pub const CHUNK: u64 = 4096; + +unsafe extern "C" { + fn racer_gcm_host_crc64(data: *const u8, length: u64) -> u64; + + fn racer_gcm_host_aes256(key: *const [u8; 32], input: *const [u8; 16], out: *mut [u8; 16]); + + fn racer_gcm_host_decrypt( + key: *const [u8; 32], + nonce: *const [u8; 12], + aad: *const u8, + aad_len: u64, + sealed: *const u8, + n: u64, + check_crc: u32, + expected_crc: u64, + out: *mut u8, + verdict: *mut RawVerdict, + ) -> c_int; +} + +/// CRC-64/XZ through the chunked combine path the kernels use. +pub fn crc64(data: &[u8]) -> u64 { + unsafe { racer_gcm_host_crc64(data.as_ptr(), data.len() as u64) } +} + +/// One AES-256 block encryption. +pub fn aes256(key: &[u8; 32], block: &[u8; 16]) -> [u8; 16] { + let mut out = [0; 16]; + unsafe { racer_gcm_host_aes256(key, block, &mut out) }; + out +} + +/// Verify `sealed` (ciphertext then 16-byte tag) and decrypt it into `out` +/// with the derived AES-256-GCM key and nonce. `out` must be exactly the +/// ciphertext length and is written only when the verdict passes. +/// +/// Returns `None` if the lengths do not match or scratch memory is unavailable. +pub fn decrypt( + key: &[u8; 32], + nonce: &[u8; 12], + aad: &[u8], + sealed: &[u8], + expected_crc: Option, + out: &mut [u8], +) -> Option { + if sealed.len().checked_sub(16) != Some(out.len()) { + return None; + } + let mut verdict = RawVerdict::default(); + let rc = unsafe { + racer_gcm_host_decrypt( + key, + nonce, + aad.as_ptr(), + aad.len() as u64, + sealed.as_ptr(), + out.len() as u64, + expected_crc.is_some() as u32, + expected_crc.unwrap_or(0), + out.as_mut_ptr(), + &mut verdict, + ) + }; + (rc == 0).then_some(verdict) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + /// FIPS-197 appendix C.3 AES-256 known answer. + fn aes256_known_answer() { + let key: [u8; 32] = std::array::from_fn(|i| i as u8); + let block: [u8; 16] = std::array::from_fn(|i| (i as u8) * 0x11); + assert_eq!( + aes256(&key, &block), + [ + 0x8e, 0xa2, 0xb7, 0xca, 0x51, 0x67, 0x45, 0xbf, 0xea, 0xfc, 0x49, 0x90, 0x4b, 0x49, + 0x60, 0x89 + ] + ); + } + + #[test] + /// The CRC-64/XZ check value, and empty input. + fn crc64_check_value() { + assert_eq!(crc64(b"123456789"), 0x995d_c9bb_df19_39fa); + assert_eq!(crc64(b""), 0); + } + + #[test] + /// NIST GCM test case 14: AES-256, zero key, one zero block, no AAD. + fn gcm_nist_test_case_14() { + let ciphertext = [ + 0xce, 0xa7, 0x40, 0x3d, 0x4d, 0x60, 0x6b, 0x6e, 0x07, 0x4e, 0xc5, 0xd3, 0xba, 0xf3, + 0x9d, 0x18, + ]; + let tag = [ + 0xd0, 0xd1, 0xc8, 0xa7, 0x99, 0x99, 0x6b, 0xf0, 0x26, 0x5b, 0x98, 0xb5, 0xd4, 0x8a, + 0xb9, 0x19, + ]; + let sealed = [ciphertext, tag].concat(); + let mut out = [1; 16]; + let verdict = decrypt(&[0; 32], &[0; 12], &[], &sealed, None, &mut out).unwrap(); + assert_eq!(verdict.status, VERDICT_PASS); + assert_eq!(out, [0; 16]); + } + + #[test] + /// Mismatched output length is refused before any work. + fn decrypt_rejects_length_mismatch() { + let mut out = [0; 3]; + assert!(decrypt(&[0; 32], &[0; 12], &[], &[0; 20], None, &mut out).is_none()); + assert!(decrypt(&[0; 32], &[0; 12], &[], &[0; 2], None, &mut []).is_none()); + } +} diff --git a/cmd/racer-dataplane/gpu/src/lib.rs b/cmd/racer-dataplane/gpu/src/lib.rs new file mode 100644 index 0000000000..39b3e9a548 --- /dev/null +++ b/cmd/racer-dataplane/gpu/src/lib.rs @@ -0,0 +1,1104 @@ +//! CUDA support for Racer GPUDirect reads (`designs/racer-gpudirect.md`). +//! +//! Every CUDA, NVML and cuFile call goes through a small C shim, +//! `libracer_gpu_cuda.so.1` (`native/gpu_cuda.c`), loaded with `dlopen` and +//! checked with `racer_gpu_abi()`. All types here are `!Send`: build and use +//! them on the dataplane's native helper thread, never on I/O threads. The +//! one exception is [`FileReader`], which runs cuFile reads that +//! [`GpuFile::detach`] prepared on other threads, so slow disk reads do not +//! hold up the helper. +//! +//! # Basics +//! +//! 1. [`Gpu::load`] loads the backend and initializes CUDA. +//! 2. [`Gpu::devices`] lists GPUs; [`DeviceInfo::usable`] skips GPUs in +//! `EXCLUSIVE_PROCESS` or `PROHIBITED` compute mode. +//! 3. [`Gpu::context`] retains a GPU's primary context. +//! 4. [`Context::import`] maps a client allocation (a POSIX fd shareable +//! handle) for one request. Writes are limited to the request's window. +//! 5. [`Context::staging`] allocates dataplane-owned device memory for +//! ciphertext. It can be filled from the host, from disk with cuFile +//! ([`GpuFile::read`]) or by a NIC through a dma-buf ([`StagingBuffer::dma_buf`]). +//! 6. [`Stream`] queues copies and [`Stream::decrypt`], then +//! [`Stream::synchronize`] waits for them. Only then is the write visible. +//! +//! # Lifetimes and leaks +//! +//! A stream keeps every buffer it touched alive until a successful +//! synchronize, so an import is never unmapped while queued GPU work may still +//! write it. If a free, unmap or synchronize fails, the resource is leaked on +//! purpose, counted in [`Gpu::leaked`], and the shim stays loaded. +//! +//! # Backends +//! +//! - `native`: loads `libracer_gpu_cuda.so.1`. +//! - `simulation`: [`simulation::Simulation`], fake GPUs in host memory. +//! - Neither: [`Gpu::load`] returns [`Error::Unavailable`]. +//! +//! Hardware-free tests: `cargo test -p racer-gpu --features simulation`. +mod ffi; +pub mod gcm; + +#[cfg(any(test, feature = "simulation"))] +pub mod simulation; + +pub use ffi::{MAX_AAD, MAX_DECRYPT, flag}; + +use ffi::{Out, RawDecrypt, RawDevice, RawVerdict, Shared}; +use std::{ + cell::{Cell, RefCell}, + ffi::{c_int, c_void}, + ops::Range, + os::fd::{AsRawFd, FromRawFd, OwnedFd}, + ptr::NonNull, + rc::Rc, +}; + +/// Result type for this crate. +pub type Result = std::result::Result; + +/// Errors from this crate. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Error { + /// No backend, no driver, or the feature is not built in. + Unavailable, + + /// Bad arguments or handle. + InvalidRequest, + + /// An offset or length is outside a buffer or the request window. + InvalidRange, + + /// The allocation or buffer is on another GPU. + WrongDevice, + + /// A CUDA, NVML or cuFile call failed with this status. + Driver(i32), + + /// Host or device memory allocation failed. + OutOfMemory, + + /// The shim was built without decrypt kernels, or they failed to load. + KernelUnavailable, + + /// A cuFile read failed or was short. + Io, + + /// Wrong state: work in flight, or no verdict to read. + Busy, + + /// The call failed and a resource was leaked on purpose. + Leaked, +} + +impl std::fmt::Display for Error { + /// Describe the error. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Driver(code) => write!(f, "GPU driver call failed with status {code}"), + other => write!(f, "{other:?}"), + } + } +} + +impl std::error::Error for Error {} + +/// A GPU's compute mode (`CUcomputemode`). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ComputeMode { + /// Several contexts may share the GPU. + Default, + + /// No contexts allowed. + Prohibited, + + /// One process at a time. + ExclusiveProcess, + + /// Unknown, or the query failed. + Other(i32), +} + +impl ComputeMode { + /// From the raw `CUcomputemode` value. + fn from_raw(value: i32) -> Self { + match value { + 0 => Self::Default, + 2 => Self::Prohibited, + 3 => Self::ExclusiveProcess, + other => Self::Other(other), + } + } +} + +/// One GPU from discovery. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DeviceInfo { + /// CUDA device ordinal. + pub ordinal: i32, + + /// GPU UUID. + pub uuid: [u8; 16], + + /// Lowercase PCI address, for example `0000:3b:00.0`. + pub pci_bus_id: String, + + /// NUMA node from NVML or sysfs, if known. + pub numa_node: Option, + + /// Compute mode. + pub compute_mode: ComputeMode, + + /// `flag::*` capability bits. + pub flags: u32, + + /// Device memory in bytes. + pub total_memory: u64, +} + +impl DeviceInfo { + /// Whether the dataplane can use this GPU: default compute mode, and it + /// can import POSIX fd handles into virtual memory. + pub fn usable(&self) -> bool { + self.compute_mode == ComputeMode::Default + && self.flags & (flag::VMM | flag::POSIX_FD) == flag::VMM | flag::POSIX_FD + } + + /// The UUID in `GPU-xxxxxxxx-xxxx-...` form, as `nvidia-smi` prints it. + pub fn uuid_string(&self) -> String { + let hex: String = self.uuid.iter().map(|b| format!("{b:02x}")).collect(); + format!( + "GPU-{}-{}-{}-{}-{}", + &hex[..8], + &hex[8..12], + &hex[12..16], + &hex[16..20], + &hex[20..] + ) + } + + /// From the C record. + fn from_raw(raw: &RawDevice) -> Self { + let end = raw.pci_bus_id.iter().position(|&b| b == 0).unwrap_or(32); + Self { + ordinal: raw.ordinal, + uuid: raw.uuid, + pci_bus_id: String::from_utf8_lossy(&raw.pci_bus_id[..end]).into_owned(), + numa_node: u32::try_from(raw.numa_node).ok(), + compute_mode: ComputeMode::from_raw(raw.compute_mode), + flags: raw.flags, + total_memory: raw.total_memory, + } + } +} + +/// Call a shim function that writes a handle; return it or the error. +fn create(shared: &Shared, call: impl FnOnce(Out) -> i32) -> Result> { + let mut out = std::ptr::null_mut(); + shared.check(call(&mut out))?; + NonNull::new(out).ok_or(Error::InvalidRequest) +} + +/// A loaded GPU backend with CUDA initialized. +pub struct Gpu { + shared: Rc, +} + +impl Gpu { + /// Load the backend and initialize CUDA. Inside a + /// [`simulation::Simulation::enter`] scope this uses the simulation. + pub fn load() -> Result { + #[cfg(any(test, feature = "simulation"))] + let api = match simulation::current() { + Some(_) => simulation::api(), + None => ffi::Api::load()?, + }; + #[cfg(not(any(test, feature = "simulation")))] + let api = ffi::Api::load()?; + let shared = Rc::new(Shared { + api, + leaks: Cell::new(0), + }); + shared.check(unsafe { (shared.api.init)() })?; + Ok(Self { shared }) + } + + /// Every GPU, including ones that are not [`DeviceInfo::usable`]. + pub fn devices(&self) -> Result> { + let mut raw = vec![RawDevice::default(); 64]; + let count = unsafe { (self.shared.api.discover)(raw.as_mut_ptr(), raw.len() as u32) }; + self.shared.check(count.min(0))?; + let count = count as usize; + if count > raw.len() { + return Err(Error::InvalidRequest); + } + Ok(raw[..count].iter().map(DeviceInfo::from_raw).collect()) + } + + /// The usable GPUs only. + pub fn usable_devices(&self) -> Result> { + Ok(self + .devices()? + .into_iter() + .filter(DeviceInfo::usable) + .collect()) + } + + /// Retain a GPU's primary context. + pub fn context(&self, ordinal: i32) -> Result { + let shared = &self.shared; + let raw = create(shared, |out| unsafe { (shared.api.context)(ordinal, out) })?; + Ok(Context(Rc::new(ContextInner { + shared: shared.clone(), + raw, + ordinal, + }))) + } + + /// Open the cuFile driver if needed and register an `O_DIRECT` file. + pub fn open_file(&self, fd: OwnedFd) -> Result { + let shared = &self.shared; + shared.check(unsafe { (shared.api.file_open)() })?; + let fd = cufile_descriptor(fd); + let raw = create(shared, |out| unsafe { + (shared.api.file)(fd.as_raw_fd(), out) + })?; + Ok(GpuFile { + shared: shared.clone(), + raw, + fd: Some(fd), + }) + } + + /// Resources leaked on purpose since this backend was loaded. + pub fn leaked(&self) -> u64 { + self.shared.leaks.get() + } + + /// A handle that runs [`FileRead`]s on any thread. It does not keep the + /// backend loaded: drop every reader thread's work before this [`Gpu`] + /// and the objects the reads name. + pub fn file_reader(&self) -> FileReader { + FileReader { + file_open: self.shared.api.file_open, + file_read: self.shared.api.file_read, + last_driver_error: self.shared.api.last_driver_error, + #[cfg(any(test, feature = "simulation"))] + simulation: simulation::current(), + } + } +} + +/// Runs cuFile reads off the helper thread. Cheap to clone. +#[derive(Clone)] +pub struct FileReader { + file_open: unsafe extern "C" fn() -> c_int, + + file_read: unsafe extern "C" fn(*mut c_void, *mut c_void, u64, u64, u64) -> i64, + + last_driver_error: unsafe extern "C" fn() -> i32, + + /// The simulation the backend was loaded in, entered on reader threads. + #[cfg(any(test, feature = "simulation"))] + simulation: Option, +} + +impl FileReader { + /// Open the cuFile driver, as [`Gpu::open_file`] does on first use. + /// Opening probes every GPU and can take most of a second, so callers + /// may do it ahead of time on another thread. A failed open is final. + pub fn open_driver(&self) -> Result<()> { + #[cfg(any(test, feature = "simulation"))] + let _environment = self.enter(); + ffi::check(unsafe { (self.file_open)() }, self.last_driver_error) + } + + /// Run `read` and wait for it. Errors are those of [`GpuFile::read`]. + pub fn read(&self, read: FileRead) -> Result<()> { + #[cfg(any(test, feature = "simulation"))] + let _environment = self.enter(); + let n = unsafe { + (self.file_read)( + read.file.as_ptr(), + read.staging.as_ptr(), + read.offset, + read.file_offset, + read.length, + ) + }; + if n < 0 { + return ffi::check(n as i32, self.last_driver_error); + } + if n as u64 != read.length { + return Err(Error::Io); + } + Ok(()) + } + + /// Enter the backend's simulation if this thread is not in one. + #[cfg(any(test, feature = "simulation"))] + fn enter(&self) -> Option { + match (&self.simulation, simulation::current()) { + (Some(simulation), None) => Some(simulation.enter()), + _ => None, + } + } +} + +/// A cuFile read made by [`GpuFile::detach`], to run with [`FileReader::read`]. +#[derive(Debug)] +pub struct FileRead { + file: NonNull, + + staging: NonNull, + + offset: u64, + + file_offset: u64, + + length: u64, +} + +// SAFETY: the shim's cuFile read is thread safe. `GpuFile::detach` makes the +// caller keep the file and staging buffer alive until the read is done. +unsafe impl Send for FileRead {} + +/// A retained primary context. Clones share it; it is released when the last +/// clone and every object made from it are gone. +#[derive(Clone)] +pub struct Context(Rc); + +/// The owned context handle. +struct ContextInner { + shared: Rc, + + raw: NonNull, + + ordinal: i32, +} + +impl Drop for ContextInner { + /// Release the context, or leak it if that fails. + fn drop(&mut self) { + if unsafe { (self.shared.api.context_free)(self.raw.as_ptr()) } != 0 { + self.shared.leak(); + } + } +} + +impl Context { + /// The CUDA device ordinal. + pub fn ordinal(&self) -> i32 { + self.0.ordinal + } + + /// Leaks seen by this context's backend. + pub fn leaked(&self) -> u64 { + self.0.shared.leaks.get() + } + + /// Create a non-blocking stream. + pub fn stream(&self) -> Result { + let shared = &self.0.shared; + let raw = create(shared, |out| unsafe { + (shared.api.stream)(self.0.raw.as_ptr(), out) + })?; + Ok(Stream { + context: self.clone(), + raw, + retained: RefCell::new(Vec::new()), + }) + } + + /// Import a client allocation and map all `allocation_size` bytes + /// read-write. Writes are then allowed only inside `window`, the + /// request's `[offset, offset + length)`. Keeps `fd` until the import is + /// released, then closes it. + pub fn import( + &self, + fd: OwnedFd, + allocation_size: u64, + window: Range, + ) -> Result { + if window.start >= window.end || window.end > allocation_size { + return Err(Error::InvalidRange); + } + let shared = &self.0.shared; + let raw = create(shared, |out| unsafe { + (shared.api.import)(self.0.raw.as_ptr(), fd.as_raw_fd(), allocation_size, out) + })?; + Ok(ImportedAllocation(Rc::new(ImportInner { + context: self.clone(), + raw, + size: allocation_size, + window, + fd: Cell::new(Some(fd)), + }))) + } + + /// Allocate `size` bytes of staging memory (rounded up to 2 MiB on the + /// device; offsets are checked against `size`). + pub fn staging(&self, size: u64) -> Result { + let shared = &self.0.shared; + let raw = create(shared, |out| unsafe { + (shared.api.staging)(self.0.raw.as_ptr(), size, out) + })?; + Ok(StagingBuffer(Rc::new(StagingInner { + context: self.clone(), + raw, + size, + registered: Cell::new(false), + in_flight: Cell::new(0), + }))) + } +} + +/// A client allocation mapped into the dataplane's context for one request. +pub struct ImportedAllocation(Rc); + +/// The mapping. Freed when the handle and every stream using it let go. +struct ImportInner { + context: Context, + + raw: NonNull, + + size: u64, + + window: Range, + + fd: Cell>, +} + +impl Drop for ImportInner { + /// Unmap and release, then close the fd. On failure leak everything, + /// including the fd and the context. + fn drop(&mut self) { + let shared = &self.context.0.shared; + if unsafe { (shared.api.import_free)(self.raw.as_ptr()) } != 0 { + shared.leak(); + std::mem::forget(self.fd.take()); + std::mem::forget(self.context.clone()); + } + } +} + +impl ImportedAllocation { + /// The size passed to `cuMemCreate`. + pub fn size(&self) -> u64 { + self.0.size + } + + /// The writable window. + pub fn window(&self) -> Range { + self.0.window.clone() + } + + /// The GPU it lives on. + pub fn ordinal(&self) -> i32 { + self.0.context.ordinal() + } +} + +/// Dataplane-owned device memory for ciphertext. +pub struct StagingBuffer(Rc); + +/// The allocation. +struct StagingInner { + context: Context, + + raw: NonNull, + + size: u64, + + registered: Cell, + + /// Streams holding it until their next synchronize. + in_flight: Cell, +} + +impl Drop for StagingInner { + /// Deregister from cuFile, then free. On failure leak. + fn drop(&mut self) { + let shared = &self.context.0.shared; + if self.registered.get() + && unsafe { (shared.api.staging_unregister)(self.raw.as_ptr()) } != 0 + { + shared.leak(); + std::mem::forget(self.context.clone()); + return; + } + if unsafe { (shared.api.staging_free)(self.raw.as_ptr()) } != 0 { + shared.leak(); + std::mem::forget(self.context.clone()); + } + } +} + +impl StagingBuffer { + /// Usable bytes. + pub fn size(&self) -> u64 { + self.0.size + } + + /// Whether a stream may still be using it. + pub fn in_flight(&self) -> bool { + self.0.in_flight.get() > 0 + } + + /// Register with cuFile so reads go straight to it. + pub fn register_cufile(&self) -> Result<()> { + if self.0.registered.get() { + return Ok(()); + } + let shared = &self.0.context.0.shared; + shared.check(unsafe { (shared.api.file_open)() })?; + shared.check(unsafe { (shared.api.staging_register)(self.0.raw.as_ptr()) })?; + self.0.registered.set(true); + Ok(()) + } + + /// Export `[offset, offset + length)` as a dma-buf fd, for + /// `ibv_reg_dmabuf_mr`. Both must be multiples of the host page size. + pub fn dma_buf(&self, offset: u64, length: u64) -> Result { + let shared = &self.0.context.0.shared; + let mut fd = -1; + shared + .check(unsafe { (shared.api.dmabuf)(self.0.raw.as_ptr(), offset, length, &mut fd) })?; + if fd < 0 { + return Err(Error::InvalidRequest); + } + Ok(unsafe { OwnedFd::from_raw_fd(fd) }) + } +} + +/// Where a stream writes. +#[derive(Clone, Copy)] +pub enum Destination<'a> { + /// A client allocation, limited to its window. + Import(&'a ImportedAllocation), + + /// Dataplane staging memory. + Staging(&'a StagingBuffer), +} + +/// Something a stream keeps alive until synchronized. +enum Retained { + Import(#[allow(dead_code)] Rc), + + Staging(Rc), + + /// Host bytes a queued copy may still read. + Host(#[allow(dead_code)] Box>), +} + +impl Drop for Retained { + /// Release one in-flight hold on staging memory. + fn drop(&mut self) { + if let Self::Staging(inner) = self { + inner.in_flight.set(inner.in_flight.get() - 1); + } + } +} + +/// One page to verify and decrypt on the GPU. +pub struct DecryptRequest<'a> { + /// XAES-derived AES-256-GCM key (`racer_crypto::DerivedPageKey::key`). + pub key: &'a [u8; 32], + + /// XAES-derived GCM nonce (`racer_crypto::DerivedPageKey::nonce`). + pub nonce: &'a [u8; 12], + + /// The page AAD (`security::page_aad`). At most [`MAX_AAD`] bytes. + pub aad: &'a [u8], + + /// Plaintext length. The source holds this many bytes plus the tag. + pub length: u64, + + /// Stored CRC-64/XZ of the sealed page, if known. + pub expected_crc: Option, +} + +/// The outcome of a GPU decrypt. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Verdict { + /// Verified and written. `crc` is the sealed page's CRC-64/XZ. + Passed { + /// CRC-64/XZ of the sealed page. + crc: u64, + }, + + /// The CRC did not match. Nothing was written. + CrcMismatch { + /// The CRC the GPU computed. + crc: u64, + }, + + /// The tag did not match. Nothing was written. + TagMismatch { + /// The CRC the GPU computed. + crc: u64, + }, + + /// Verification did not run (a launch failed). Nothing was written. + NotRun, +} + +/// A CUDA stream. Work is visible only after [`Stream::synchronize`]. +pub struct Stream { + context: Context, + + raw: NonNull, + + retained: RefCell>, +} + +impl Stream { + /// The backend state. + fn shared(&self) -> &Rc { + &self.context.0.shared + } + + /// Check a destination range and return its raw handle and hold. + fn destination( + &self, + dst: Destination<'_>, + offset: u64, + length: u64, + ) -> Result<(*mut c_void, Retained)> { + let end = offset.checked_add(length).ok_or(Error::InvalidRange)?; + match dst { + Destination::Import(import) => { + let inner = &import.0; + if !Rc::ptr_eq(&inner.context.0, &self.context.0) { + return Err(Error::WrongDevice); + } + if offset < inner.window.start || end > inner.window.end { + return Err(Error::InvalidRange); + } + Ok((inner.raw.as_ptr(), Retained::Import(inner.clone()))) + } + Destination::Staging(staging) => self.source(staging, offset, length), + } + } + + /// Check a staging range and return its raw handle and hold. + fn source( + &self, + staging: &StagingBuffer, + offset: u64, + length: u64, + ) -> Result<(*mut c_void, Retained)> { + let inner = &staging.0; + if !Rc::ptr_eq(&inner.context.0, &self.context.0) { + return Err(Error::WrongDevice); + } + let end = offset.checked_add(length).ok_or(Error::InvalidRange)?; + if end > inner.size { + return Err(Error::InvalidRange); + } + inner.in_flight.set(inner.in_flight.get() + 1); + Ok((inner.raw.as_ptr(), Retained::Staging(inner.clone()))) + } + + /// Keep holds until the next synchronize. Held even when the call + /// failed, because part of it may have been queued. + fn retain(&self, holds: impl IntoIterator) { + self.retained.borrow_mut().extend(holds); + } + + /// Queue a copy of host bytes. CUDA may keep reading page-locked host + /// memory after the call returns, so the stream takes ownership of + /// `bytes` and holds it until [`Stream::synchronize`] succeeds (or leaks + /// it if the stream cannot be synchronized). Pass a cheap shared handle, + /// such as an `Arc`, to avoid copying large buffers. + pub fn copy_from_host(&self, dst: Destination<'_>, offset: u64, bytes: S) -> Result<()> + where + S: AsRef<[u8]> + 'static, + { + // Box first so the bytes do not move after the pointer is taken. + let source: Box> = Box::new(bytes); + let slice = (*source).as_ref(); + let (ptr, length) = (slice.as_ptr(), slice.len() as u64); + if length == 0 { + return Err(Error::InvalidRange); + } + let (raw, hold) = self.destination(dst, offset, length)?; + let rc = unsafe { + (self.shared().api.copy_h2d)(self.raw.as_ptr(), raw, offset, ptr.cast(), length) + }; + self.retain([hold, Retained::Host(source)]); + self.shared().check(rc) + } + + /// Number of host sources held for queued copies. Test helper. + #[doc(hidden)] + pub fn retained_host_sources(&self) -> usize { + self.retained + .borrow() + .iter() + .filter(|hold| matches!(hold, Retained::Host(_))) + .count() + } + + /// Queue a device copy from staging memory. + pub fn copy_from_staging( + &self, + dst: Destination<'_>, + dst_offset: u64, + src: &StagingBuffer, + src_offset: u64, + length: u64, + ) -> Result<()> { + if length == 0 { + return Err(Error::InvalidRange); + } + let (raw_dst, dst_hold) = self.destination(dst, dst_offset, length)?; + let (raw_src, src_hold) = self.source(src, src_offset, length)?; + let rc = unsafe { + (self.shared().api.copy_d2d)( + self.raw.as_ptr(), + raw_dst, + dst_offset, + raw_src, + src_offset, + length, + ) + }; + self.retain([dst_hold, src_hold]); + self.shared().check(rc) + } + + /// Copy staging bytes to the host and wait for this stream. Client + /// imports can never be read. + pub fn read_staging(&self, src: &StagingBuffer, offset: u64, out: &mut [u8]) -> Result<()> { + if out.is_empty() { + return Err(Error::InvalidRange); + } + let (raw, hold) = self.source(src, offset, out.len() as u64)?; + let rc = unsafe { + (self.shared().api.copy_d2h)( + self.raw.as_ptr(), + out.as_mut_ptr().cast(), + raw, + offset, + out.len() as u64, + ) + }; + self.retain([hold]); + self.shared().check(rc)?; + // copy_d2h synchronized the stream. + self.retained.borrow_mut().clear(); + Ok(()) + } + + /// Queue verify-then-decrypt of the sealed page at `src_offset` into + /// `dst` at `dst_offset`. The destination is written only if the checks + /// pass, decided on the GPU. Read the result with [`Stream::verdict`] + /// after [`Stream::synchronize`]. One decrypt per synchronize; a + /// second one first returns [`Error::Busy`]. + pub fn decrypt( + &self, + request: &DecryptRequest<'_>, + src: &StagingBuffer, + src_offset: u64, + dst: Destination<'_>, + dst_offset: u64, + ) -> Result<()> { + if request.length > MAX_DECRYPT || request.aad.len() > MAX_AAD { + return Err(Error::InvalidRequest); + } + let mut holds = Vec::with_capacity(2); + let (raw_src, hold) = self.source(src, src_offset, request.length + 16)?; + holds.push(hold); + // A zero-length page writes nothing but still needs a destination + // on this GPU inside the window. + let (raw_dst, hold) = self.destination(dst, dst_offset, request.length)?; + holds.push(hold); + let raw = RawDecrypt { + key: *request.key, + nonce: *request.nonce, + check_crc: request.expected_crc.is_some() as u32, + expected_crc: request.expected_crc.unwrap_or(0), + length: request.length, + aad: request.aad.as_ptr(), + aad_length: request.aad.len() as u64, + }; + let rc = unsafe { + (self.shared().api.decrypt)( + self.raw.as_ptr(), + &raw, + raw_src, + src_offset, + raw_dst, + dst_offset, + ) + }; + drop(raw); + self.retain(holds); + self.shared().check(rc) + } + + /// Wait for all queued work, then release holds on the buffers it used. + pub fn synchronize(&self) -> Result<()> { + self.shared() + .check(unsafe { (self.shared().api.stream_sync)(self.raw.as_ptr()) })?; + self.retained.borrow_mut().clear(); + Ok(()) + } + + /// The verdict of the last decrypt, after [`Stream::synchronize`]. + pub fn verdict(&self) -> Result { + let mut raw = RawVerdict::default(); + self.shared() + .check(unsafe { (self.shared().api.verdict)(self.raw.as_ptr(), &mut raw) })?; + Ok(match raw.status { + gcm::VERDICT_PASS => Verdict::Passed { crc: raw.crc }, + gcm::VERDICT_CRC => Verdict::CrcMismatch { crc: raw.crc }, + gcm::VERDICT_TAG => Verdict::TagMismatch { crc: raw.crc }, + _ => Verdict::NotRun, + }) + } +} + +impl Drop for Stream { + /// Synchronize and free. If that fails, leak the stream and every buffer + /// it may still be using. + fn drop(&mut self) { + let shared = self.context.0.shared.clone(); + if unsafe { (shared.api.stream_free)(self.raw.as_ptr()) } != 0 { + shared.leak(); + std::mem::forget(std::mem::take(&mut *self.retained.borrow_mut())); + std::mem::forget(self.context.clone()); + } + } +} + +/// Return a descriptor cuFile accepts for the same open file. +/// +/// cuFile refuses to register a descriptor whose status flags include +/// `O_NOFOLLOW` (`CU_FILE_INVALID_FILE_OPEN_FLAG`, 5019), and Racer opens every +/// slab file and block device with it. Reopening `/proc/self/fd/N` follows the +/// kernel's link to the already-open file, not a path, so dropping the flag +/// here does not reintroduce the symlink race it guards against. The new +/// descriptor keeps the access mode and `O_DIRECT`, and is used only if it +/// names the same file. Otherwise the original is returned unchanged and +/// cuFile reports its own error. +fn cufile_descriptor(fd: OwnedFd) -> OwnedFd { + let raw = fd.as_raw_fd(); + let flags = unsafe { libc::fcntl(raw, libc::F_GETFL) }; + if flags < 0 || flags & libc::O_NOFOLLOW == 0 { + return fd; + } + let path = format!("/proc/self/fd/{raw}\0"); + let reopen = (flags & !(libc::O_NOFOLLOW | libc::O_CREAT | libc::O_EXCL | libc::O_TRUNC)) + | libc::O_CLOEXEC; + let new = unsafe { libc::open(path.as_ptr().cast(), reopen) }; + if new < 0 { + return fd; + } + let new = unsafe { OwnedFd::from_raw_fd(new) }; + match (stat(&fd), stat(&new)) { + (Some(a), Some(b)) + if a.st_dev == b.st_dev && a.st_ino == b.st_ino && a.st_rdev == b.st_rdev => + { + new + } + _ => fd, + } +} + +fn stat(fd: &OwnedFd) -> Option { + let mut out = std::mem::MaybeUninit::::uninit(); + if unsafe { libc::fstat(fd.as_raw_fd(), out.as_mut_ptr()) } != 0 { + return None; + } + Some(unsafe { out.assume_init() }) +} + +/// A file registered with cuFile. +pub struct GpuFile { + shared: Rc, + + raw: NonNull, + + fd: Option, +} + +impl GpuFile { + /// Read `length` bytes at `file_offset` into staging at `offset`. Waits + /// for the read. The staging buffer must not be in use by a stream. + pub fn read( + &self, + staging: &StagingBuffer, + offset: u64, + file_offset: u64, + length: u64, + ) -> Result<()> { + if staging.in_flight() { + return Err(Error::Busy); + } + let end = offset.checked_add(length).ok_or(Error::InvalidRange)?; + if length == 0 || end > staging.size() { + return Err(Error::InvalidRange); + } + let n = unsafe { + (self.shared.api.file_read)( + self.raw.as_ptr(), + staging.0.raw.as_ptr(), + offset, + file_offset, + length, + ) + }; + if n < 0 { + return self.shared.check(n as i32); + } + if n as u64 != length { + return Err(Error::Io); + } + Ok(()) + } + + /// Prepare a read of `length` bytes at `file_offset` into staging at + /// `offset`, to run on another thread with [`FileReader::read`]. Unlike + /// [`GpuFile::read`], streams may use other parts of the staging buffer + /// meanwhile. + /// + /// # Safety + /// + /// Until the read returns, the caller must keep this file, `staging` and + /// the [`Gpu`] alive, and must not let anything else touch + /// `[offset, offset + length)` of `staging`. + pub unsafe fn detach( + &self, + staging: &StagingBuffer, + offset: u64, + file_offset: u64, + length: u64, + ) -> Result { + let end = offset.checked_add(length).ok_or(Error::InvalidRange)?; + if length == 0 || end > staging.size() { + return Err(Error::InvalidRange); + } + Ok(FileRead { + file: self.raw, + staging: staging.0.raw, + offset, + file_offset, + length, + }) + } +} + +impl Drop for GpuFile { + /// Deregister, then close the fd. On failure leak both. + fn drop(&mut self) { + if unsafe { (self.shared.api.file_free)(self.raw.as_ptr()) } != 0 { + self.shared.leak(); + std::mem::forget(self.fd.take()); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn open(path: &std::path::Path, flags: i32) -> OwnedFd { + let path = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).unwrap(); + let fd = unsafe { libc::open(path.as_ptr(), flags | libc::O_CLOEXEC) }; + assert!(fd >= 0, "open: {}", std::io::Error::last_os_error()); + unsafe { OwnedFd::from_raw_fd(fd) } + } + + fn status(fd: &OwnedFd) -> i32 { + unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFL) } + } + + fn temp_file(name: &str) -> std::path::PathBuf { + let path = std::env::temp_dir().join(format!("racer-gpu-{}-{name}", std::process::id())); + std::fs::write(&path, b"racer").unwrap(); + path + } + + #[test] + /// cuFile rejects O_NOFOLLOW, so the descriptor is reopened without it + /// and keeps the same file and access mode. + fn cufile_descriptor_drops_nofollow() { + let path = temp_file("nofollow"); + let fd = open(&path, libc::O_RDWR | libc::O_NOFOLLOW); + let before = stat(&fd).unwrap(); + let fd = cufile_descriptor(fd); + let after = stat(&fd).unwrap(); + let flags = status(&fd); + assert_eq!(flags & libc::O_NOFOLLOW, 0); + assert_eq!(flags & libc::O_ACCMODE, libc::O_RDWR); + assert_eq!((before.st_dev, before.st_ino), (after.st_dev, after.st_ino)); + let cloexec = unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFD) }; + assert_ne!(cloexec & libc::FD_CLOEXEC, 0); + std::fs::remove_file(path).unwrap(); + } + + #[test] + /// An unlinked file is still reopened through the kernel's link. + fn cufile_descriptor_follows_open_file_not_path() { + let path = temp_file("unlinked"); + let fd = open(&path, libc::O_RDONLY | libc::O_NOFOLLOW); + let before = stat(&fd).unwrap(); + std::fs::remove_file(&path).unwrap(); + std::fs::write(&path, b"other").unwrap(); + let fd = cufile_descriptor(fd); + let after = stat(&fd).unwrap(); + assert_eq!(status(&fd) & libc::O_NOFOLLOW, 0); + assert_eq!((before.st_dev, before.st_ino), (after.st_dev, after.st_ino)); + std::fs::remove_file(path).unwrap(); + } + + #[test] + /// A descriptor without O_NOFOLLOW is returned as is. + fn cufile_descriptor_keeps_other_descriptors() { + let path = temp_file("plain"); + let fd = open(&path, libc::O_RDONLY); + let raw = fd.as_raw_fd(); + let fd = cufile_descriptor(fd); + assert_eq!(fd.as_raw_fd(), raw); + std::fs::remove_file(path).unwrap(); + } + + #[test] + /// Without a backend in scope or built in, loading fails cleanly. + fn load_without_backend_is_unavailable() { + #[cfg(not(feature = "native"))] + assert_eq!(Gpu::load().err(), Some(Error::Unavailable)); + } + + #[test] + /// Compute mode and capabilities gate usability. + fn usable_requires_default_mode_and_vmm_fd_support() { + let mut device = DeviceInfo { + ordinal: 0, + uuid: [0xab; 16], + pci_bus_id: "0000:3b:00.0".into(), + numa_node: Some(1), + compute_mode: ComputeMode::Default, + flags: flag::VMM | flag::POSIX_FD, + total_memory: 1, + }; + assert!(device.usable()); + assert_eq!( + device.uuid_string(), + "GPU-abababab-abab-abab-abab-abababababab" + ); + for mode in [ + ComputeMode::Prohibited, + ComputeMode::ExclusiveProcess, + ComputeMode::Other(-1), + ] { + device.compute_mode = mode; + assert!(!device.usable()); + } + device.compute_mode = ComputeMode::Default; + device.flags = flag::VMM; + assert!(!device.usable()); + } +} diff --git a/cmd/racer-dataplane/gpu/src/simulation.rs b/cmd/racer-dataplane/gpu/src/simulation.rs new file mode 100644 index 0000000000..d35b2996ae --- /dev/null +++ b/cmd/racer-dataplane/gpu/src/simulation.rs @@ -0,0 +1,972 @@ +//! Fake GPUs for tests. No CUDA, driver or hardware needed. +//! +//! It implements the same C functions as `native/gpu_cuda.c`, so the real +//! wrappers in `lib.rs` run on top of it unchanged. Device memory is host +//! memory; a client allocation is a memfd made with +//! [`Simulation::allocate`] and passed as the shareable handle. Decryption +//! runs the serial host emulation of the kernels (`native/gcm_host.c`). +//! +//! Work runs when queued, not on synchronize, so tests check ordering with +//! [`Simulation::trace`] instead of timing. +//! +//! # Basics +//! +//! 1. Create a [`Simulation`] with [`SimDevice`]s and [`Simulation::enter`] it. +//! 2. [`crate::Gpu::load`] now uses it. +//! 3. Script one-shot failures with [`Simulation::fault`]. +//! 4. Check live resources with [`Simulation::live`]. +use crate::{ + ffi::{self, Api, RawDecrypt, RawDevice, RawVerdict, code}, + gcm, +}; + +use std::{ + cell::RefCell, + collections::{BTreeMap, VecDeque}, + ffi::{c_int, c_void}, + fs::File, + os::{ + fd::{FromRawFd, OwnedFd}, + unix::fs::{FileExt, MetadataExt}, + }, + rc::Rc, + sync::{Arc, Mutex, MutexGuard}, +}; + +/// Granularity imports must be a multiple of. Real GPUs use 2 MiB; smaller +/// here keeps tests light. +pub const GRANULARITY: u64 = 64 << 10; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +/// One simulated GPU. +#[derive(Clone, Debug)] +pub struct SimDevice { + /// GPU UUID. + pub uuid: [u8; 16], + + /// Lowercase PCI address. + pub pci_bus_id: String, + + /// NUMA node, if known. + pub numa_node: Option, + + /// Raw `CUcomputemode`. + pub compute_mode: i32, + + /// `flag::*` bits. + pub flags: u32, + + /// Device memory. + pub total_memory: u64, +} + +impl SimDevice { + /// A usable GPU with every capability. + pub fn new(index: u8) -> Self { + Self { + uuid: [index; 16], + pci_bus_id: format!("0000:{:02x}:00.0", 0x10 + index), + numa_node: Some(index as u32 % 2), + compute_mode: 0, + flags: ffi::flag::VMM | ffi::flag::POSIX_FD | ffi::flag::DMA_BUF | ffi::flag::GDR, + total_memory: 80 << 30, + } + } +} + +/// A shim function. Used in traces and to target faults. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Operation { + Init, + Discover, + Context, + ContextFree, + Stream, + StreamSync, + StreamFree, + Import, + ImportFree, + Staging, + StagingFree, + CopyH2d, + CopyD2d, + CopyD2h, + Decrypt, + Verdict, + FileOpen, + File, + FileFree, + StagingRegister, + StagingUnregister, + FileRead, + Dmabuf, +} + +/// One call in the trace. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Event { + /// The function. + pub operation: Operation, + + /// The handle it acted on or created, 0 if none. + pub handle: usize, + + /// Its result code. + pub result: i64, +} + +/// Count of live resources of each kind. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct Live { + pub contexts: usize, + pub streams: usize, + pub imports: usize, + pub staging: usize, + pub files: usize, +} + +/// A simulated set of GPUs, shared by clones. Deterministic. +#[derive(Clone)] +pub struct Simulation { + world: Arc>, +} + +/// Scope guard from [`Simulation::enter`]. Restores the previous one on drop. +pub struct Environment { + previous: Option, +} + +impl Drop for Environment { + /// Put back the previous simulation. + fn drop(&mut self) { + CURRENT.with(|current| *current.borrow_mut() = self.previous.take()); + } +} + +/// A retained context. +struct SimContext { + ordinal: i32, +} + +/// A stream and its last verdict. +struct SimStream { + context: usize, + + /// 0 none, 1 queued, 2 synchronized, as in the shim. + decrypt: u8, + + verdict: RawVerdict, +} + +/// Device memory: client memfd or staging bytes. +enum Memory { + Import { file: File }, + Staging { bytes: Mapped, registered: bool }, +} + +/// Staging bytes in a shared memfd mapping. A dma-buf export of the whole +/// buffer is a duplicate of the memfd, so a simulated NIC that maps it writes +/// bytes the simulated GPU then reads, as with real GPUDirect RDMA. +struct Mapped { + file: File, + pointer: std::ptr::NonNull, + length: usize, +} + +// SAFETY: the mapping is owned and only reached through the world's lock. +unsafe impl Send for Mapped {} + +impl Mapped { + /// A zeroed shared mapping of `length` bytes, or None. + fn new(length: usize) -> Option { + let fd = unsafe { libc::memfd_create(c"racer-gpu-staging".as_ptr(), libc::MFD_CLOEXEC) }; + if fd < 0 { + return None; + } + let file = unsafe { File::from_raw_fd(fd) }; + file.set_len(length as u64).ok()?; + let pointer = unsafe { + libc::mmap( + std::ptr::null_mut(), + length, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + std::os::fd::AsRawFd::as_raw_fd(&file), + 0, + ) + }; + if pointer == libc::MAP_FAILED { + return None; + } + Some(Self { + file, + pointer: std::ptr::NonNull::new(pointer.cast())?, + length, + }) + } +} + +impl std::ops::Deref for Mapped { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + // SAFETY: the mapping lives as long as self. Other writers (a + // simulated NIC) only write while the dataplane holds the slot. + unsafe { std::slice::from_raw_parts(self.pointer.as_ptr(), self.length) } + } +} + +impl std::ops::DerefMut for Mapped { + fn deref_mut(&mut self) -> &mut [u8] { + // SAFETY: as for deref. + unsafe { std::slice::from_raw_parts_mut(self.pointer.as_ptr(), self.length) } + } +} + +impl Drop for Mapped { + fn drop(&mut self) { + unsafe { libc::munmap(self.pointer.as_ptr().cast(), self.length) }; + } +} + +/// One buffer. +struct Buffer { + context: usize, + + size: u64, + + memory: Memory, +} + +/// Everything the simulation tracks. +#[derive(Default)] +struct World { + devices: Vec, + + next: usize, + + contexts: BTreeMap, + + streams: BTreeMap, + + buffers: BTreeMap, + + files: BTreeMap, + + /// memfd inode to device ordinal. + allocations: BTreeMap<(u64, u64), i32>, + + faults: VecDeque<(Operation, c_int)>, + + trace: Vec, + + initialized: bool, + + file_driver: bool, + + no_kernels: bool, + + last_error: i32, +} + +impl Simulation { + /// The world, locked. A panic on another thread does not poison it for + /// the rest of a test. + fn world(&self) -> MutexGuard<'_, World> { + self.world + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + /// GPUs that discovery will report, in ordinal order. + pub fn new(devices: Vec) -> Self { + Self { + world: Arc::new(Mutex::new(World { + devices, + next: 0x1000, + ..World::default() + })), + } + } + + /// Make this the simulation [`crate::Gpu::load`] uses on this thread. + pub fn enter(&self) -> Environment { + let previous = CURRENT.with(|current| current.borrow_mut().replace(self.clone())); + Environment { previous } + } + + /// A client allocation of `size` bytes on GPU `ordinal`, as the fd + /// `cuMemExportToShareableHandle` would return. Keep a clone to read it. + pub fn allocate(&self, ordinal: i32, size: u64) -> OwnedFd { + let fd = unsafe { libc::memfd_create(c"racer-gpu-sim".as_ptr(), libc::MFD_CLOEXEC) }; + assert!(fd >= 0, "memfd_create failed"); + let file = unsafe { File::from_raw_fd(fd) }; + file.set_len(size).expect("size memfd"); + let meta = file.metadata().expect("stat memfd"); + self.world() + .allocations + .insert((meta.dev(), meta.ino()), ordinal); + file.into() + } + + /// Fail the next call to `operation` with `code` (a `gpu_cuda.h` code; + /// `-5` is a driver error). Nothing changes. + pub fn fault(&self, operation: Operation, code: c_int) { + self.world().faults.push_back((operation, code)); + } + + /// Behave like a shim built without decrypt kernels. + pub fn disable_kernels(&self) { + self.world().no_kernels = true; + } + + /// Every call so far. + pub fn trace(&self) -> Vec { + self.world().trace.clone() + } + + /// Live resources. + pub fn live(&self) -> Live { + let world = self.world(); + let (mut imports, mut staging) = (0, 0); + for buffer in world.buffers.values() { + match buffer.memory { + Memory::Import { .. } => imports += 1, + Memory::Staging { .. } => staging += 1, + } + } + Live { + contexts: world.contexts.len(), + streams: world.streams.len(), + imports, + staging, + files: world.files.len(), + } + } +} + +/// The simulation in scope on this thread. +pub(crate) fn current() -> Option { + CURRENT.with(|current| current.borrow().clone()) +} + +/// Run `f` on the current world, recording the call and applying a fault. +fn with + Copy + From>( + operation: Operation, + f: impl FnOnce(&mut World) -> (R, usize), +) -> R { + let Some(simulation) = current() else { + return R::from(code::EUNAVAILABLE); + }; + let mut world = simulation.world(); + let fault = world + .faults + .iter() + .position(|(op, _)| *op == operation) + .and_then(|i| world.faults.remove(i)); + let (result, handle) = match fault { + Some((_, rc)) => { + if rc == code::EDRIVER { + world.last_error = 999; + } + (R::from(rc), 0) + } + None => f(&mut world), + }; + world.trace.push(Event { + operation, + handle, + result: result.into(), + }); + result +} + +/// Store a new handle. +fn give(world: &mut World, out: ffi::Out) -> usize { + world.next += 0x10; + unsafe { *out = std::ptr::without_provenance_mut(world.next) }; + world.next +} + +/// Handle value of an opaque pointer. +fn id(pointer: *mut c_void) -> usize { + pointer.addr() +} + +/// Nonempty `[offset, offset + length)` inside `size`. +fn span(size: u64, offset: u64, length: u64) -> bool { + length > 0 && offset <= size && length <= size - offset +} + +unsafe extern "C" fn last_driver_error() -> i32 { + current().map_or(0, |s| s.world().last_error) +} + +unsafe extern "C" fn init() -> c_int { + with(Operation::Init, |w| { + w.initialized = true; + (code::OK, 0) + }) +} + +unsafe extern "C" fn discover(out: *mut RawDevice, capacity: u32) -> c_int { + with(Operation::Discover, |w| { + if !w.initialized { + return (code::EUNAVAILABLE, 0); + } + for (i, device) in w.devices.iter().enumerate().take(capacity as usize) { + let mut raw = RawDevice { + uuid: device.uuid, + ordinal: i as i32, + numa_node: device.numa_node.map_or(-1, |n| n as i32), + compute_mode: device.compute_mode, + flags: device.flags, + total_memory: device.total_memory, + ..RawDevice::default() + }; + let bytes = device.pci_bus_id.as_bytes(); + raw.pci_bus_id[..bytes.len().min(31)].copy_from_slice(&bytes[..bytes.len().min(31)]); + unsafe { *out.add(i) = raw }; + } + (w.devices.len() as c_int, 0) + }) +} + +unsafe extern "C" fn context(ordinal: i32, out: ffi::Out) -> c_int { + with(Operation::Context, |w| { + if ordinal < 0 || ordinal as usize >= w.devices.len() { + w.last_error = 101; // CUDA_ERROR_INVALID_DEVICE + return (code::EDRIVER, 0); + } + let handle = give(w, out); + w.contexts.insert(handle, SimContext { ordinal }); + (code::OK, handle) + }) +} + +unsafe extern "C" fn context_free(context: *mut c_void) -> c_int { + with(Operation::ContextFree, |w| { + let handle = id(context); + let used = w.streams.values().any(|s| s.context == handle) + || w.buffers.values().any(|b| b.context == handle); + if used { + // Rust keeps the context alive while anything uses it; refuse + // so a bug shows up as a counted leak. + return (code::EBUSY, handle); + } + match w.contexts.remove(&handle) { + Some(_) => (code::OK, handle), + None => (code::EINVAL, handle), + } + }) +} + +unsafe extern "C" fn stream(context: *mut c_void, out: ffi::Out) -> c_int { + with(Operation::Stream, |w| { + if !w.contexts.contains_key(&id(context)) { + return (code::EINVAL, 0); + } + let handle = give(w, out); + w.streams.insert( + handle, + SimStream { + context: id(context), + decrypt: 0, + verdict: RawVerdict::default(), + }, + ); + (code::OK, handle) + }) +} + +unsafe extern "C" fn stream_sync(stream: *mut c_void) -> c_int { + with(Operation::StreamSync, |w| { + match w.streams.get_mut(&id(stream)) { + Some(s) => { + if s.decrypt == 1 { + s.decrypt = 2; + } + (code::OK, id(stream)) + } + None => (code::EINVAL, id(stream)), + } + }) +} + +unsafe extern "C" fn stream_free(stream: *mut c_void) -> c_int { + with(Operation::StreamFree, |w| { + match w.streams.remove(&id(stream)) { + Some(_) => (code::OK, id(stream)), + None => (code::EINVAL, id(stream)), + } + }) +} + +unsafe extern "C" fn import(context: *mut c_void, fd: c_int, size: u64, out: ffi::Out) -> c_int { + with(Operation::Import, |w| { + let Some(ctx) = w.contexts.get(&id(context)) else { + return (code::EINVAL, 0); + }; + let ordinal = ctx.ordinal; + if fd < 0 || size == 0 { + return (code::EINVAL, 0); + } + // Like cuMemImportFromShareableHandle, keep our own reference. + let duplicate = unsafe { libc::fcntl(fd, libc::F_DUPFD_CLOEXEC, 0) }; + if duplicate < 0 { + w.last_error = 1; // CUDA_ERROR_INVALID_VALUE + return (code::EDRIVER, 0); + } + let file = unsafe { File::from_raw_fd(duplicate) }; + let Ok(meta) = file.metadata() else { + return (code::EDRIVER, 0); + }; + let Some(&owner) = w.allocations.get(&(meta.dev(), meta.ino())) else { + w.last_error = 1; + return (code::EDRIVER, 0); + }; + if owner != ordinal { + return (code::EDEVICE, 0); + } + if !size.is_multiple_of(GRANULARITY) { + return (code::EINVAL, 0); + } + if size > meta.len() { + w.last_error = 1; + return (code::EDRIVER, 0); + } + let handle = give(w, out); + w.buffers.insert( + handle, + Buffer { + context: id(context), + size, + memory: Memory::Import { file }, + }, + ); + (code::OK, handle) + }) +} + +unsafe extern "C" fn import_free(allocation: *mut c_void) -> c_int { + with(Operation::ImportFree, |w| { + let handle = id(allocation); + match w.buffers.get(&handle) { + Some(Buffer { + memory: Memory::Import { .. }, + .. + }) => { + w.buffers.remove(&handle); + (code::OK, handle) + } + _ => (code::EINVAL, handle), + } + }) +} + +unsafe extern "C" fn staging(context: *mut c_void, size: u64, out: ffi::Out) -> c_int { + with(Operation::Staging, |w| { + if !w.contexts.contains_key(&id(context)) || size == 0 { + return (code::EINVAL, 0); + } + if size > 1 << 32 { + return (code::ENOMEM, 0); + } + let Some(bytes) = Mapped::new(size as usize) else { + return (code::ENOMEM, 0); + }; + let handle = give(w, out); + w.buffers.insert( + handle, + Buffer { + context: id(context), + size, + memory: Memory::Staging { + bytes, + registered: false, + }, + }, + ); + (code::OK, handle) + }) +} + +unsafe extern "C" fn staging_free(staging: *mut c_void) -> c_int { + with(Operation::StagingFree, |w| { + let handle = id(staging); + match w.buffers.get(&handle) { + Some(Buffer { + memory: Memory::Staging { registered, .. }, + .. + }) => { + if *registered { + return (code::EBUSY, handle); + } + w.buffers.remove(&handle); + (code::OK, handle) + } + _ => (code::EINVAL, handle), + } + }) +} + +impl World { + /// Check that a stream and buffer exist and share a context. + fn pair(&self, stream: *mut c_void, buffer: *mut c_void) -> Result<(), c_int> { + let s = self.streams.get(&id(stream)).ok_or(code::EINVAL)?; + let b = self.buffers.get(&id(buffer)).ok_or(code::EINVAL)?; + if s.context != b.context { + return Err(code::EDEVICE); + } + Ok(()) + } + + /// Write bytes into a buffer, bounds already checked. + fn write(&mut self, buffer: usize, offset: u64, bytes: &[u8]) -> c_int { + match &mut self.buffers.get_mut(&buffer).expect("checked").memory { + Memory::Import { file } => match file.write_all_at(bytes, offset) { + Ok(()) => code::OK, + Err(_) => code::EDRIVER, + }, + Memory::Staging { bytes: memory, .. } => { + memory[offset as usize..offset as usize + bytes.len()].copy_from_slice(bytes); + code::OK + } + } + } + + /// Staging bytes, or None for an import. + fn staging(&self, buffer: usize) -> Option<&[u8]> { + match &self.buffers.get(&buffer)?.memory { + Memory::Staging { bytes, .. } => Some(bytes), + Memory::Import { .. } => None, + } + } +} + +unsafe extern "C" fn copy_h2d( + stream: *mut c_void, + dst: *mut c_void, + offset: u64, + src: *const c_void, + length: u64, +) -> c_int { + with(Operation::CopyH2d, |w| { + if let Err(rc) = w.pair(stream, dst) { + return (rc, id(dst)); + } + if src.is_null() || !span(w.buffers[&id(dst)].size, offset, length) { + return (code::ERANGE, id(dst)); + } + let bytes = unsafe { std::slice::from_raw_parts(src.cast::(), length as usize) }; + (w.write(id(dst), offset, bytes), id(dst)) + }) +} + +unsafe extern "C" fn copy_d2d( + stream: *mut c_void, + dst: *mut c_void, + dst_offset: u64, + src: *mut c_void, + src_offset: u64, + length: u64, +) -> c_int { + with(Operation::CopyD2d, |w| { + if let Err(rc) = w.pair(stream, dst).and(w.pair(stream, src)) { + return (rc, id(dst)); + } + let Some(from) = w.staging(id(src)) else { + return (code::EINVAL, id(dst)); + }; + if !span(w.buffers[&id(dst)].size, dst_offset, length) + || !span(from.len() as u64, src_offset, length) + { + return (code::ERANGE, id(dst)); + } + let bytes = from[src_offset as usize..(src_offset + length) as usize].to_vec(); + (w.write(id(dst), dst_offset, &bytes), id(dst)) + }) +} + +unsafe extern "C" fn copy_d2h( + stream: *mut c_void, + dst: *mut c_void, + src: *mut c_void, + src_offset: u64, + length: u64, +) -> c_int { + with(Operation::CopyD2h, |w| { + if let Err(rc) = w.pair(stream, src) { + return (rc, id(src)); + } + let Some(from) = w.staging(id(src)) else { + return (code::EINVAL, id(src)); + }; + if dst.is_null() || !span(from.len() as u64, src_offset, length) { + return (code::ERANGE, id(src)); + } + let out = unsafe { std::slice::from_raw_parts_mut(dst.cast::(), length as usize) }; + out.copy_from_slice(&from[src_offset as usize..(src_offset + length) as usize]); + let s = w.streams.get_mut(&id(stream)).expect("checked"); + if s.decrypt == 1 { + s.decrypt = 2; + } + (code::OK, id(src)) + }) +} + +unsafe extern "C" fn decrypt( + stream: *mut c_void, + params: *const RawDecrypt, + src: *mut c_void, + src_offset: u64, + dst: *mut c_void, + dst_offset: u64, +) -> c_int { + with(Operation::Decrypt, |w| { + if params.is_null() { + return (code::EINVAL, 0); + } + let p = unsafe { &*params }; + if let Err(rc) = w.pair(stream, dst).and(w.pair(stream, src)) { + return (rc, id(dst)); + } + if p.length > ffi::MAX_DECRYPT + || p.aad_length > ffi::MAX_AAD as u64 + || (p.aad.is_null() && p.aad_length > 0) + || p.check_crc > 1 + { + return (code::EINVAL, id(dst)); + } + let Some(from) = w.staging(id(src)) else { + return (code::EINVAL, id(dst)); + }; + let n = p.length; + let dst_size = w.buffers[&id(dst)].size; + if !span(from.len() as u64, src_offset, n + 16) + || (n > 0 && !span(dst_size, dst_offset, n)) + || (n == 0 && dst_offset > dst_size) + { + return (code::ERANGE, id(dst)); + } + if w.streams[&id(stream)].decrypt == 1 { + return (code::EBUSY, id(dst)); + } + if w.no_kernels { + return (code::ENOKERNEL, id(dst)); + } + let sealed = &from[src_offset as usize..(src_offset + n + 16) as usize]; + let aad = if p.aad_length == 0 { + &[][..] + } else { + unsafe { std::slice::from_raw_parts(p.aad, p.aad_length as usize) } + }; + let mut out = zeroize::Zeroizing::new(vec![0; n as usize]); + let expected = (p.check_crc == 1).then_some(p.expected_crc); + let Some(verdict) = gcm::decrypt(&p.key, &p.nonce, aad, sealed, expected, &mut out) else { + return (code::ENOMEM, id(dst)); + }; + let mut rc = code::OK; + if verdict.status == gcm::VERDICT_PASS && n > 0 { + rc = w.write(id(dst), dst_offset, &out); + } + let s = w.streams.get_mut(&id(stream)).expect("checked"); + s.decrypt = 1; + s.verdict = verdict; + (rc, id(dst)) + }) +} + +unsafe extern "C" fn verdict(stream: *mut c_void, out: *mut RawVerdict) -> c_int { + with(Operation::Verdict, |w| { + let Some(s) = w.streams.get_mut(&id(stream)) else { + return (code::EINVAL, 0); + }; + if s.decrypt != 2 || out.is_null() { + return (code::EBUSY, id(stream)); + } + unsafe { *out = std::mem::take(&mut s.verdict) }; + s.decrypt = 0; + (code::OK, id(stream)) + }) +} + +unsafe extern "C" fn file_open() -> c_int { + with(Operation::FileOpen, |w| { + w.file_driver = true; + (code::OK, 0) + }) +} + +unsafe extern "C" fn file(fd: c_int, out: ffi::Out) -> c_int { + with(Operation::File, |w| { + if !w.file_driver { + return (code::EUNAVAILABLE, 0); + } + if fd < 0 || unsafe { libc::fcntl(fd, libc::F_GETFD) } < 0 { + return (code::EINVAL, 0); + } + // cuFile reads with its own alignment rules. Reopen the same file + // without O_DIRECT so host reads of any length and buffer work. + let Ok(reopened) = File::open(format!("/proc/self/fd/{fd}")) else { + return (code::EINVAL, 0); + }; + let handle = give(w, out); + w.files.insert(handle, reopened); + (code::OK, handle) + }) +} + +unsafe extern "C" fn file_free(file: *mut c_void) -> c_int { + with(Operation::FileFree, |w| match w.files.remove(&id(file)) { + Some(_) => (code::OK, id(file)), + None => (code::EINVAL, id(file)), + }) +} + +/// Set a staging buffer's cuFile registration. +fn register(w: &mut World, staging: *mut c_void, value: bool) -> (c_int, usize) { + if !w.file_driver { + return (code::EUNAVAILABLE, id(staging)); + } + match w.buffers.get_mut(&id(staging)).map(|b| &mut b.memory) { + Some(Memory::Staging { registered, .. }) if *registered != value => { + *registered = value; + (code::OK, id(staging)) + } + Some(Memory::Staging { .. }) => (code::EBUSY, id(staging)), + _ => (code::EINVAL, id(staging)), + } +} + +unsafe extern "C" fn staging_register(staging: *mut c_void) -> c_int { + with(Operation::StagingRegister, |w| register(w, staging, true)) +} + +unsafe extern "C" fn staging_unregister(staging: *mut c_void) -> c_int { + with(Operation::StagingUnregister, |w| { + register(w, staging, false) + }) +} + +unsafe extern "C" fn file_read( + file: *mut c_void, + staging: *mut c_void, + offset: u64, + file_offset: u64, + length: u64, +) -> i64 { + with(Operation::FileRead, |w| { + let Some(source) = w.files.get(&id(file)) else { + return (code::EINVAL as i64, 0); + }; + let Some(Buffer { + memory: Memory::Staging { bytes, .. }, + size, + .. + }) = w.buffers.get_mut(&id(staging)) + else { + return (code::EINVAL as i64, 0); + }; + if !span(*size, offset, length) { + return (code::ERANGE as i64, id(staging)); + } + let out = &mut bytes[offset as usize..(offset + length) as usize]; + match source.read_exact_at(out, file_offset) { + Ok(()) => (length as i64, id(staging)), + Err(_) => (code::EIO as i64, id(staging)), + } + }) +} + +unsafe extern "C" fn dmabuf( + staging: *mut c_void, + offset: u64, + length: u64, + fd: *mut c_int, +) -> c_int { + with(Operation::Dmabuf, |w| { + let Some(Buffer { + memory: Memory::Staging { bytes, .. }, + .. + }) = w.buffers.get(&id(staging)) + else { + return (code::EINVAL, id(staging)); + }; + let size = bytes.len() as u64; + let live = (offset == 0).then(|| bytes.file.try_clone()); + if fd.is_null() { + return (code::EINVAL, id(staging)); + } + let page = 4096; + if length == 0 + || !offset.is_multiple_of(page) + || !length.is_multiple_of(page) + || offset + length > size + { + return (code::ERANGE, id(staging)); + } + // An export from offset 0 is a live view: a duplicate of the backing + // memfd, so the dma-buf offset equals the staging offset as it does on + // real hardware. Other offsets get a placeholder fd of the right size. + if let Some(file) = live { + let Ok(file) = file else { + return (code::EDRIVER, id(staging)); + }; + unsafe { *fd = std::os::fd::IntoRawFd::into_raw_fd(file) }; + return (code::OK, id(staging)); + } + let memfd = unsafe { libc::memfd_create(c"racer-gpu-dmabuf".as_ptr(), libc::MFD_CLOEXEC) }; + if memfd < 0 { + return (code::EDRIVER, id(staging)); + } + let file = unsafe { File::from_raw_fd(memfd) }; + if file.set_len(length).is_err() { + return (code::EDRIVER, id(staging)); + } + unsafe { *fd = std::os::fd::IntoRawFd::into_raw_fd(file) }; + (code::OK, id(staging)) + }) +} + +/// The simulation's function table. +pub(crate) fn api() -> Rc { + Rc::new(Api { + library: None, + last_driver_error, + init, + discover, + context, + context_free, + stream, + stream_sync, + stream_free, + import, + import_free, + staging, + staging_free, + copy_h2d, + copy_d2d, + copy_d2h, + decrypt, + verdict, + file_open, + file, + file_free, + staging_register, + staging_unregister, + file_read, + dmabuf, + }) +} + +/// Read `length` bytes at `offset` of a client allocation fd. +pub fn read_allocation(fd: &OwnedFd, offset: u64, length: usize) -> Vec { + let file = File::from(fd.try_clone().expect("dup")); + let mut out = vec![0; length]; + file.read_exact_at(&mut out, offset).expect("read memfd"); + out +} diff --git a/cmd/racer-dataplane/gpu/tests/gcm_core.rs b/cmd/racer-dataplane/gpu/tests/gcm_core.rs new file mode 100644 index 0000000000..0d97bfa134 --- /dev/null +++ b/cmd/racer-dataplane/gpu/tests/gcm_core.rs @@ -0,0 +1,186 @@ +//! Checks the C decrypt core (`native/gcm_core.h`, run serially through the +//! same chunked GHASH and CRC decomposition the CUDA kernels use) against the +//! `racer-crypto` XAES-256-GCM implementation and CRC-64/XZ. + +use racer_gpu::gcm::{self, VERDICT_CRC, VERDICT_PASS, VERDICT_TAG}; + +/// Deterministic xorshift bytes. +struct Rng(u64); + +impl Rng { + /// Fill `out` with pseudo-random bytes. + fn fill(&mut self, out: &mut [u8]) { + for chunk in out.chunks_mut(8) { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + chunk.copy_from_slice(&self.0.to_le_bytes()[..chunk.len()]); + } + } + + /// A random array. + fn array(&mut self) -> [u8; N] { + let mut out = [0; N]; + self.fill(&mut out); + out + } +} + +/// A page sealed the way the dataplane seals pages. +struct Page { + key: [u8; 32], + nonce: [u8; 24], + aad: Vec, + plaintext: Vec, + sealed: Vec, +} + +/// Seal `length` random bytes with XAES-256-GCM and a page-shaped AAD. +fn page(rng: &mut Rng, length: usize) -> Page { + let key = rng.array(); + let nonce = rng.array(); + // Same shape as security::page_aad: domain, cipher, length-prefixed + // fields, page number, key ID, nonce and both lengths. + let mut aad = b"racer/page/aead/v1\0".to_vec(); + aad.extend_from_slice(&1u32.to_be_bytes()); + aad.extend_from_slice(&36u32.to_be_bytes()); + aad.extend_from_slice(b"33333333-3333-4333-8333-333333333333"); + aad.extend_from_slice(&rng.array::<32>()); + aad.extend_from_slice(&4u32.to_be_bytes()); + aad.extend_from_slice(b"\"v1\""); + aad.extend_from_slice(&7u64.to_be_bytes()); + aad.extend_from_slice(&rng.array::<16>()); + aad.extend_from_slice(&nonce); + aad.extend_from_slice(&(length as u32).to_be_bytes()); + aad.extend_from_slice(&(length as u32 + 16).to_be_bytes()); + let mut plaintext = vec![0; length]; + rng.fill(&mut plaintext); + let mut sealed = vec![0; length + racer_crypto::TAG_LEN]; + racer_crypto::seal(&key, &nonce, &aad, &plaintext, &mut sealed).unwrap(); + Page { + key, + nonce, + aad, + plaintext, + sealed, + } +} + +/// Decrypt through the C core with the Rust-derived XAES key. +fn open(page: &Page, aad: &[u8], sealed: &[u8], crc: Option) -> (u32, u64, Vec) { + let derived = racer_crypto::xaes_derive(&page.key, &page.nonce); + let mut out = vec![0xa5; sealed.len() - 16]; + let verdict = gcm::decrypt(derived.key(), derived.nonce(), aad, sealed, crc, &mut out).unwrap(); + (verdict.status, verdict.crc, out) +} + +#[test] +/// Lengths around block and chunk edges, including empty and 16 MiB pages. +fn core_matches_crypto_crate_for_many_lengths() { + let mut rng = Rng(0x1234_5678_9abc_def1); + let chunk = gcm::CHUNK as usize; + let mut lengths = vec![ + 0, + 1, + 15, + 16, + 17, + 31, + 33, + 255, + 4079, + 4080, + 4081, + chunk - 1, + chunk, + chunk + 1, + 3 * chunk - 17, + 65536 + 5, + 1 << 20, + (1 << 20) + 13, + 16 << 20, + ]; + for _ in 0..8 { + let mut b = [0u8; 4]; + rng.fill(&mut b); + lengths.push(u32::from_le_bytes(b) as usize % 200_000); + } + for length in lengths { + let page = page(&mut rng, length); + let expected_crc = racer_crypto::crc64(&page.sealed); + assert_eq!(gcm::crc64(&page.sealed), expected_crc, "length {length}"); + let (status, crc, out) = open(&page, &page.aad, &page.sealed, Some(expected_crc)); + assert_eq!(status, VERDICT_PASS, "length {length}"); + assert_eq!(crc, expected_crc, "length {length}"); + assert!(out == page.plaintext, "length {length}"); + let (status, crc, _) = open(&page, &page.aad, &page.sealed, None); + assert_eq!( + (status, crc), + (VERDICT_PASS, expected_crc), + "length {length}" + ); + } +} + +#[test] +/// Any tamper fails verification and leaves the output untouched. +fn core_rejects_tampering_without_writing() { + let mut rng = Rng(42); + for length in [1, 16, 4097, 70_001] { + let page = page(&mut rng, length); + let crc = racer_crypto::crc64(&page.sealed); + let untouched = vec![0xa5; length]; + + // Flipped ciphertext bit, with and without a matching stored CRC. + let mut sealed = page.sealed.clone(); + sealed[length / 2] ^= 0x10; + let (status, actual, out) = open(&page, &page.aad, &sealed, Some(crc)); + assert_eq!(status, VERDICT_CRC); + assert_eq!(actual, racer_crypto::crc64(&sealed)); + assert_eq!(out, untouched); + let fresh = racer_crypto::crc64(&sealed); + let (status, _, out) = open(&page, &page.aad, &sealed, Some(fresh)); + assert_eq!(status, VERDICT_TAG); + assert_eq!(out, untouched); + + // Flipped tag bit. + let mut sealed = page.sealed.clone(); + sealed[length + 15] ^= 1; + let (status, _, out) = open(&page, &page.aad, &sealed, None); + assert_eq!(status, VERDICT_TAG); + assert_eq!(out, untouched); + + // Wrong AAD. + let mut aad = page.aad.clone(); + aad[20] ^= 1; + let (status, _, out) = open(&page, &aad, &page.sealed, Some(crc)); + assert_eq!(status, VERDICT_TAG); + assert_eq!(out, untouched); + + // Truncated AAD. + let (status, _, _) = open(&page, &page.aad[..page.aad.len() - 1], &page.sealed, None); + assert_eq!(status, VERDICT_TAG); + + // Wrong expected CRC with intact data. + let (status, actual, out) = open(&page, &page.aad, &page.sealed, Some(crc ^ 1)); + assert_eq!((status, actual), (VERDICT_CRC, crc)); + assert_eq!(out, untouched); + + // Wrong key. + let mut other = page.key; + other[0] ^= 1; + let derived = racer_crypto::xaes_derive(&other, &page.nonce); + let mut out = untouched.clone(); + let verdict = gcm::decrypt( + derived.key(), + derived.nonce(), + &page.aad, + &page.sealed, + None, + &mut out, + ) + .unwrap(); + assert_eq!(verdict.status, VERDICT_TAG); + assert_eq!(out, untouched); + } +} diff --git a/cmd/racer-dataplane/gpu/tests/simulation.rs b/cmd/racer-dataplane/gpu/tests/simulation.rs new file mode 100644 index 0000000000..56fd76ac81 --- /dev/null +++ b/cmd/racer-dataplane/gpu/tests/simulation.rs @@ -0,0 +1,571 @@ +//! Safe API behavior on the simulation backend: discovery, import windows, +//! decrypt verdicts, lifetimes, fault injection and leak counting. +#![cfg(feature = "simulation")] + +use racer_gpu::{ + DecryptRequest, Destination, Error, Gpu, Verdict, + simulation::{GRANULARITY, Operation, SimDevice, Simulation, read_allocation}, +}; +use std::os::fd::OwnedFd; + +/// Two usable GPUs and one in EXCLUSIVE_PROCESS mode. +fn simulation() -> Simulation { + let mut exclusive = SimDevice::new(2); + exclusive.compute_mode = 3; + Simulation::new(vec![SimDevice::new(0), SimDevice::new(1), exclusive]) +} + +/// A memfd with arbitrary bytes, standing in for an O_DIRECT file. +fn file_with(bytes: &[u8]) -> OwnedFd { + use std::os::unix::fs::FileExt; + let fd = unsafe { libc::memfd_create(c"file".as_ptr(), libc::MFD_CLOEXEC) }; + assert!(fd >= 0); + let file = unsafe { ::from_raw_fd(fd) }; + file.write_all_at(bytes, 0).unwrap(); + file.into() +} + +/// A sealed page and what the GPU needs to open it. +struct Page { + derived: racer_crypto::DerivedPageKey, + aad: Vec, + plaintext: Vec, + sealed: Vec, +} + +/// Seal `length` bytes with XAES-256-GCM. +fn page(length: usize) -> Page { + let key = [9; 32]; + let nonce = *b"0123456789abcdefghijklmn"; + let aad = b"racer/page/aead/v1\0test".to_vec(); + let plaintext: Vec = (0..length).map(|i| (i * 7 + 3) as u8).collect(); + let mut sealed = vec![0; length + 16]; + racer_crypto::seal(&key, &nonce, &aad, &plaintext, &mut sealed).unwrap(); + Page { + derived: racer_crypto::xaes_derive(&key, &nonce), + aad, + plaintext, + sealed, + } +} + +impl Page { + /// The GPU request. + fn request(&self, expected_crc: Option) -> DecryptRequest<'_> { + DecryptRequest { + key: self.derived.key(), + nonce: self.derived.nonce(), + aad: &self.aad, + length: self.plaintext.len() as u64, + expected_crc, + } + } +} + +#[test] +/// Discovery reports every GPU; only default-mode GPUs are usable. +fn discovery_reports_all_and_filters_unusable() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let devices = gpu.devices().unwrap(); + assert_eq!(devices.len(), 3); + assert_eq!(devices[1].pci_bus_id, "0000:11:00.0"); + assert_eq!(devices[1].numa_node, Some(1)); + let usable: Vec<_> = gpu + .usable_devices() + .unwrap() + .iter() + .map(|d| d.ordinal) + .collect(); + assert_eq!(usable, [0, 1]); +} + +#[test] +/// Init and context failures surface the driver status. +fn faults_surface_driver_errors() { + let sim = simulation(); + let _env = sim.enter(); + sim.fault(Operation::Init, -3); + assert_eq!(Gpu::load().err(), Some(Error::Unavailable)); + let gpu = Gpu::load().unwrap(); + sim.fault(Operation::Context, -5); + assert_eq!(gpu.context(0).err(), Some(Error::Driver(999))); + assert_eq!(gpu.context(7).err(), Some(Error::Driver(101))); + assert_eq!(sim.live().contexts, 0); +} + +#[test] +/// Writes are limited to the request window and the import's GPU. +fn import_rejects_out_of_window_and_wrong_device() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let other = gpu.context(1).unwrap(); + let size = 2 * GRANULARITY; + let client = sim.allocate(0, size); + + // Bad windows, granularity, and devices fail before or at import. + let dup = || client.try_clone().unwrap(); + assert_eq!( + ctx.import(dup(), size, 10..10).err(), + Some(Error::InvalidRange) + ); + assert_eq!( + ctx.import(dup(), size, 0..size + 1).err(), + Some(Error::InvalidRange) + ); + assert_eq!( + ctx.import(dup(), size - 1, 0..1).err(), + Some(Error::InvalidRequest) + ); + assert_eq!( + ctx.import(dup(), 4 * GRANULARITY, 0..1).err(), + Some(Error::Driver(1)) + ); + assert_eq!( + other.import(dup(), size, 0..1).err(), + Some(Error::WrongDevice) + ); + let stranger = file_with(&[0; 16]); + assert_eq!( + ctx.import(stranger, size, 0..1).err(), + Some(Error::Driver(1)) + ); + + let import = ctx.import(dup(), size, 100..200).unwrap(); + assert_eq!(import.window(), 100..200); + let stream = ctx.stream().unwrap(); + let foreign = other.stream().unwrap(); + let dst = Destination::Import(&import); + stream.copy_from_host(dst, 100, [7; 100]).unwrap(); + assert_eq!( + stream.copy_from_host(dst, 99, [1; 2]), + Err(Error::InvalidRange) + ); + assert_eq!( + stream.copy_from_host(dst, 150, [1; 51]), + Err(Error::InvalidRange) + ); + assert_eq!( + stream.copy_from_host(dst, u64::MAX, [1]), + Err(Error::InvalidRange) + ); + assert_eq!( + stream.copy_from_host(dst, 100, []), + Err(Error::InvalidRange) + ); + assert_eq!( + foreign.copy_from_host(dst, 100, [1]), + Err(Error::WrongDevice) + ); + stream.synchronize().unwrap(); + let bytes = read_allocation(&client, 98, 104); + assert_eq!(&bytes[..2], &[0, 0]); + assert!(bytes[2..102].iter().all(|&b| b == 7)); + assert_eq!(&bytes[102..], &[0, 0]); +} + +#[test] +/// Ciphertext staged on the GPU decrypts into the client window; tampered +/// pages and wrong CRCs leave it untouched. +fn decrypt_into_import_verifies_before_writing() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(1).unwrap(); + let client = sim.allocate(1, GRANULARITY); + let import = ctx + .import(client.try_clone().unwrap(), GRANULARITY, 64..64 + 5000) + .unwrap(); + let staging = ctx.staging(8192).unwrap(); + let stream = ctx.stream().unwrap(); + let page = page(5000); + let crc = racer_crypto::crc64(&page.sealed); + let dst = Destination::Import(&import); + + assert_eq!(stream.verdict(), Err(Error::Busy)); + stream + .copy_from_host(Destination::Staging(&staging), 16, page.sealed.clone()) + .unwrap(); + stream + .decrypt(&page.request(Some(crc)), &staging, 16, dst, 64) + .unwrap(); + // One decrypt per synchronize. + assert_eq!( + stream.decrypt(&page.request(None), &staging, 16, dst, 64), + Err(Error::Busy) + ); + stream.synchronize().unwrap(); + assert_eq!(stream.verdict(), Ok(Verdict::Passed { crc })); + assert_eq!(stream.verdict(), Err(Error::Busy)); + assert_eq!(read_allocation(&client, 64, 5000), page.plaintext); + + // Fresh destination for the failure cases. + let client2 = sim.allocate(1, GRANULARITY); + let import2 = ctx + .import(client2.try_clone().unwrap(), GRANULARITY, 0..5000) + .unwrap(); + let dst2 = Destination::Import(&import2); + stream + .decrypt(&page.request(Some(crc ^ 1)), &staging, 16, dst2, 0) + .unwrap(); + stream.synchronize().unwrap(); + assert_eq!(stream.verdict(), Ok(Verdict::CrcMismatch { crc })); + + let mut tampered = page.sealed.clone(); + tampered[4999] ^= 1; + stream + .copy_from_host(Destination::Staging(&staging), 16, tampered) + .unwrap(); + stream + .decrypt(&page.request(None), &staging, 16, dst2, 0) + .unwrap(); + stream.synchronize().unwrap(); + assert!(matches!(stream.verdict(), Ok(Verdict::TagMismatch { .. }))); + assert!(read_allocation(&client2, 0, 5000).iter().all(|&b| b == 0)); + + // Out-of-window destination and out-of-range source are refused. + assert_eq!( + stream.decrypt(&page.request(None), &staging, 16, dst2, 1), + Err(Error::InvalidRange) + ); + assert_eq!( + stream.decrypt(&page.request(None), &staging, 8192 - 5000, dst2, 0), + Err(Error::InvalidRange) + ); + let mut big = page.request(None); + big.length = racer_gpu::MAX_DECRYPT + 1; + assert_eq!( + stream.decrypt(&big, &staging, 0, dst2, 0), + Err(Error::InvalidRequest) + ); + let long_aad = vec![0; racer_gpu::MAX_AAD + 1]; + let mut long = page.request(None); + long.aad = &long_aad; + assert_eq!( + stream.decrypt(&long, &staging, 0, dst2, 0), + Err(Error::InvalidRequest) + ); +} + +#[test] +/// A shim without kernels reports it instead of decrypting. +fn decrypt_without_kernels_is_unavailable() { + let sim = simulation(); + sim.disable_kernels(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let staging = ctx.staging(64).unwrap(); + let out = ctx.staging(64).unwrap(); + let stream = ctx.stream().unwrap(); + let page = page(16); + assert_eq!( + stream.decrypt( + &page.request(None), + &staging, + 0, + Destination::Staging(&out), + 0 + ), + Err(Error::KernelUnavailable) + ); +} + +#[test] +/// Dropping an import while work is queued defers the unmap to after the +/// stream synchronizes. +fn import_is_released_only_after_stream_sync() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let client = sim.allocate(0, GRANULARITY); + let import = ctx.import(client, GRANULARITY, 0..10).unwrap(); + let stream = ctx.stream().unwrap(); + stream + .copy_from_host(Destination::Import(&import), 0, *b"0123456789") + .unwrap(); + drop(import); + assert_eq!(sim.live().imports, 1); + stream.synchronize().unwrap(); + assert_eq!(sim.live().imports, 0); + let order: Vec<_> = sim + .trace() + .iter() + .map(|e| e.operation) + .filter(|op| matches!(op, Operation::StreamSync | Operation::ImportFree)) + .collect(); + assert_eq!(order, [Operation::StreamSync, Operation::ImportFree]); + + // Dropping the stream itself synchronizes before releasing holds. + let client = sim.allocate(0, GRANULARITY); + let import = ctx.import(client, GRANULARITY, 0..10).unwrap(); + stream + .copy_from_host(Destination::Import(&import), 0, *b"x") + .unwrap(); + drop(import); + drop(stream); + assert_eq!(sim.live().imports, 0); + drop(ctx); + assert_eq!(sim.live().contexts, 0); + assert_eq!(gpu.leaked(), 0); +} + +#[test] +/// Failed frees leak the resource and count it; nothing is freed early. +fn failed_frees_leak_and_count() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let import = ctx + .import(sim.allocate(0, GRANULARITY), GRANULARITY, 0..1) + .unwrap(); + sim.fault(Operation::ImportFree, -5); + drop(import); + assert_eq!(gpu.leaked(), 1); + assert_eq!(sim.live().imports, 1); + + // A stream that cannot be freed keeps everything it touched. + let stream = ctx.stream().unwrap(); + let staging = ctx.staging(16).unwrap(); + stream + .copy_from_host(Destination::Staging(&staging), 0, [1; 16]) + .unwrap(); + assert!(staging.in_flight()); + sim.fault(Operation::StreamFree, -5); + drop(stream); + drop(staging); + assert_eq!(gpu.leaked(), 2); + assert_eq!(sim.live().streams, 1); + assert_eq!(sim.live().staging, 1); + + // A failed synchronize keeps holds; the buffer cannot be freed yet. + let stream = ctx.stream().unwrap(); + let staging = ctx.staging(16).unwrap(); + stream + .copy_from_host(Destination::Staging(&staging), 0, [1; 16]) + .unwrap(); + sim.fault(Operation::StreamSync, -5); + assert_eq!(stream.synchronize(), Err(Error::Driver(999))); + assert!(staging.in_flight()); + stream.synchronize().unwrap(); + assert!(!staging.in_flight()); + + // Context leaks when release fails. + let leaked_ctx = gpu.context(1).unwrap(); + sim.fault(Operation::ContextFree, -5); + drop(leaked_ctx); + assert_eq!(gpu.leaked(), 3); +} + +#[test] +/// cuFile reads land in staging, respect bounds, and wait for streams. +fn cufile_reads_into_staging() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let staging = ctx.staging(8192).unwrap(); + staging.register_cufile().unwrap(); + let data: Vec = (0..10000u32).map(|i| i as u8).collect(); + let file = gpu.open_file(file_with(&data)).unwrap(); + file.read(&staging, 100, 4096, 1000).unwrap(); + let stream = ctx.stream().unwrap(); + let mut out = vec![0; 1000]; + stream.read_staging(&staging, 100, &mut out).unwrap(); + assert_eq!(out, &data[4096..5096]); + assert_eq!(file.read(&staging, 8000, 0, 200), Err(Error::InvalidRange)); + assert_eq!(file.read(&staging, 0, 0, 0), Err(Error::InvalidRange)); + assert_eq!(file.read(&staging, 0, 9990, 100), Err(Error::Io)); + + // A queued copy holds the buffer. + stream + .copy_from_host(Destination::Staging(&staging), 0, [1]) + .unwrap(); + assert_eq!(file.read(&staging, 0, 0, 1), Err(Error::Busy)); + stream.synchronize().unwrap(); + file.read(&staging, 0, 0, 1).unwrap(); + drop(file); + drop(staging); + assert_eq!(sim.live().files, 0); + assert_eq!(sim.live().staging, 0); + assert_eq!(gpu.leaked(), 0); +} + +#[test] +/// Detached cuFile reads run on other threads, while a stream uses the same +/// staging buffer, and report errors like a direct read. +fn detached_cufile_reads_run_on_other_threads() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let staging = ctx.staging(8192).unwrap(); + staging.register_cufile().unwrap(); + let data: Vec = (0..10000u32).map(|i| (i * 7) as u8).collect(); + let file = gpu.open_file(file_with(&data)).unwrap(); + let reader = gpu.file_reader(); + + // A queued copy into the first page does not block a read of the second. + let stream = ctx.stream().unwrap(); + stream + .copy_from_host(Destination::Staging(&staging), 0, [1]) + .unwrap(); + let reads = [(4096, 0, 1000), (6000, 5000, 2000)].map(|(offset, file_offset, length)| { + unsafe { file.detach(&staging, offset, file_offset, length) }.unwrap() + }); + let threads = reads.map(|read| { + let reader = reader.clone(); + std::thread::spawn(move || reader.read(read)) + }); + for thread in threads { + thread.join().unwrap().unwrap(); + } + stream.synchronize().unwrap(); + let mut out = vec![0; 1000]; + stream.read_staging(&staging, 4096, &mut out).unwrap(); + assert_eq!(out, &data[..1000]); + let mut out = vec![0; 2000]; + stream.read_staging(&staging, 6000, &mut out).unwrap(); + assert_eq!(out, &data[5000..7000]); + + // Bounds are checked up front; I/O and driver errors come from the read. + assert_eq!( + unsafe { file.detach(&staging, 8000, 0, 200) }.err(), + Some(Error::InvalidRange) + ); + assert_eq!( + unsafe { file.detach(&staging, 0, 0, 0) }.err(), + Some(Error::InvalidRange) + ); + let short = unsafe { file.detach(&staging, 0, 9990, 100) }.unwrap(); + let thread = std::thread::spawn({ + let reader = reader.clone(); + move || reader.read(short) + }); + assert_eq!(thread.join().unwrap(), Err(Error::Io)); + sim.fault(Operation::FileRead, -5); + let faulted = unsafe { file.detach(&staging, 0, 0, 100) }.unwrap(); + let thread = std::thread::spawn(move || reader.read(faulted)); + assert!(matches!(thread.join().unwrap(), Err(Error::Driver(_)))); + drop(file); + drop(staging); + assert_eq!(sim.live().files, 0); + assert_eq!(gpu.leaked(), 0); +} + +#[test] +/// dma-buf export checks page alignment and bounds. +fn dma_buf_export_checks_alignment() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let staging = ctx.staging(16384).unwrap(); + staging.dma_buf(4096, 8192).unwrap(); + assert_eq!(staging.dma_buf(1, 4096).err(), Some(Error::InvalidRange)); + assert_eq!(staging.dma_buf(0, 100).err(), Some(Error::InvalidRange)); + assert_eq!( + staging.dma_buf(8192, 16384).err(), + Some(Error::InvalidRange) + ); +} + +#[test] +/// A whole-buffer export is a live view: writes through the dma-buf (as a +/// NIC would make) are what the GPU then reads, and the other way around. +fn dma_buf_export_from_zero_is_live() { + use std::os::unix::fs::FileExt; + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let staging = ctx.staging(16384).unwrap(); + let file = std::fs::File::from(staging.dma_buf(0, 16384).unwrap()); + file.write_all_at(b"from the nic", 8192).unwrap(); + let stream = ctx.stream().unwrap(); + let mut out = [0; 12]; + stream.read_staging(&staging, 8192, &mut out).unwrap(); + stream.synchronize().unwrap(); + assert_eq!(&out, b"from the nic"); + stream + .copy_from_host(Destination::Staging(&staging), 4096, *b"gpu") + .unwrap(); + stream.synchronize().unwrap(); + let mut back = [0; 3]; + file.read_exact_at(&mut back, 4096).unwrap(); + assert_eq!(&back, b"gpu"); + drop((stream, staging, file)); + assert_eq!(sim.live().staging, 0); +} + +#[test] +/// Device copies move staged bytes into the window; reads of staging work. +fn device_copy_from_staging() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let client = sim.allocate(0, GRANULARITY); + let import = ctx + .import(client.try_clone().unwrap(), GRANULARITY, 10..20) + .unwrap(); + let staging = ctx.staging(32).unwrap(); + let stream = ctx.stream().unwrap(); + stream + .copy_from_host(Destination::Staging(&staging), 0, *b"abcdefghij") + .unwrap(); + stream + .copy_from_staging(Destination::Import(&import), 10, &staging, 0, 10) + .unwrap(); + assert_eq!( + stream.copy_from_staging(Destination::Import(&import), 10, &staging, 0, 11), + Err(Error::InvalidRange) + ); + assert_eq!( + stream.copy_from_staging(Destination::Import(&import), 10, &staging, 30, 5), + Err(Error::InvalidRange) + ); + stream.synchronize().unwrap(); + assert_eq!(read_allocation(&client, 10, 10), b"abcdefghij"); +} + +#[test] +fn host_source_is_held_until_stream_sync() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let client = sim.allocate(0, GRANULARITY); + let import = ctx + .import(client.try_clone().unwrap(), GRANULARITY, 0..10) + .unwrap(); + let stream = ctx.stream().unwrap(); + let source = std::sync::Arc::<[u8]>::from(&b"0123456789"[..]); + let weak = std::sync::Arc::downgrade(&source); + stream + .copy_from_host(Destination::Import(&import), 0, source.clone()) + .unwrap(); + // The caller drops its handle early; the stream still owns the bytes. + drop(source); + assert_eq!(weak.strong_count(), 1); + assert_eq!(stream.retained_host_sources(), 1); + stream.synchronize().unwrap(); + assert_eq!(weak.strong_count(), 0); + assert_eq!(stream.retained_host_sources(), 0); + assert_eq!(read_allocation(&client, 0, 10), b"0123456789"); + + // A rejected copy does not keep the source. + let source = std::sync::Arc::<[u8]>::from(vec![1u8; 11]); + let weak = std::sync::Arc::downgrade(&source); + assert_eq!( + stream.copy_from_host(Destination::Import(&import), 0, source), + Err(Error::InvalidRange) + ); + assert_eq!(weak.strong_count(), 0); + assert_eq!(stream.retained_host_sources(), 0); +} diff --git a/cmd/racer-dataplane/http/src/connection.rs b/cmd/racer-dataplane/http/src/connection.rs index 52f802b351..fe28f80975 100644 --- a/cmd/racer-dataplane/http/src/connection.rs +++ b/cmd/racer-dataplane/http/src/connection.rs @@ -300,6 +300,10 @@ pub struct ConnectionLease { read_ahead: Option<(OwnedBuffer, Range)>, + /// Descriptors from the receive that produced `read_ahead`. They belong to + /// a later request head that starts inside those bytes. + ahead_descriptors: ReceivedDescriptors, + rx_remaining: Option, tx_remaining: Option, @@ -321,6 +325,50 @@ pub struct HeadCompletion { pub _decoded: Option, } +/// Most descriptors retained across one request head. Excess descriptors are +/// closed on arrival and the head is marked truncated. +pub const MAX_HEAD_DESCRIPTORS: usize = 8; + +/// Descriptors passed with `SCM_RIGHTS` while a request head was read. +#[derive(Debug, Default)] +pub struct ReceivedDescriptors { + /// Received descriptors, close-on-exec, in arrival order. + pub descriptors: Vec, + + /// Some ancillary data was truncated (`MSG_CTRUNC`) and its descriptors lost. + pub truncated: bool, +} + +impl ReceivedDescriptors { + /// Whether any descriptor or truncation was recorded. + fn has_any(&self) -> bool { + self.truncated || !self.descriptors.is_empty() + } + + /// Move `other` into `self`, closing descriptors beyond the head cap. + fn absorb(&mut self, other: ReceivedDescriptors) { + self.truncated |= other.truncated; + for descriptor in other.descriptors { + if self.descriptors.len() < MAX_HEAD_DESCRIPTORS { + self.descriptors.push(descriptor); + } else { + // Over the aggregate cap: close now and reject the head. + drop(descriptor); + self.truncated = true; + } + } + } +} + +/// A decoded request head and the descriptors that arrived with it. +pub struct RequestHead { + /// Admitted request head. + pub head: MessageHead, + + /// Descriptors received while reading this head. + pub descriptors: ReceivedDescriptors, +} + /// Fixed admitted storage, zeroized before release or single-buffer reuse. pub struct OwnedBuffer { bytes: Vec, @@ -557,7 +605,7 @@ impl HttpIo { ) -> Operation<'a, C, HeadCompletion> { Box::pin(async move { let completed = self - .receive_head_outcome(connection, scope, header_limit, false) + .receive_head_outcome(connection, scope, header_limit, false, None) .await?; Ok(HeadCompletion { connection: completed.connection, @@ -576,7 +624,33 @@ impl HttpIo { scope: &'a C::Scope, header_limit: usize, ) -> Operation<'a, C, HeadCompletion>> { - self.receive_head_outcome(connection, scope, header_limit, true) + self.receive_head_outcome(connection, scope, header_limit, true, None) + } + + /// Like [`Self::receive_request_head_limited`], but read with `recvmsg` and + /// keep any `SCM_RIGHTS` descriptors that arrived while reading the head. + /// Descriptors that came with a rejected head are closed. The caller owns + /// the returned descriptors and must close any the request does not claim. + pub fn receive_request_head_with_descriptors<'a>( + &'a self, + connection: ConnectionLease, + scope: &'a C::Scope, + header_limit: usize, + ) -> Operation<'a, C, HeadCompletion>> { + Box::pin(async move { + let sink = Rc::new(RefCell::new(ReceivedDescriptors::default())); + let completed = self + .receive_head_outcome(connection, scope, header_limit, true, Some(sink.clone())) + .await?; + let descriptors = std::mem::take(&mut *sink.borrow_mut()); + Ok(HeadCompletion { + connection: completed.connection, + value: completed + .value + .map(|head| RequestHead { head, descriptors }), + _decoded: completed._decoded, + }) + }) } /// Decode and admit a head without allowing hooks to change wire framing. @@ -586,6 +660,7 @@ impl HttpIo { scope: &'a C::Scope, header_limit: usize, request_only: bool, + ancillary: Option>>, ) -> Operation<'a, C, HeadCompletion>> { Box::pin(async move { scope.check()?; @@ -598,6 +673,20 @@ impl HttpIo { let mut buffer = self.buffer(codec.header_limit().min(4096.max(ahead_length)))?; let mut used = 0; let mut scan = HeadScan::default(); + // Descriptors of the most recent receive (or of the carried + // read-ahead). Linux unix_stream_read_generic (net/unix/af_unix.c) + // may merge several skbs into one recvmsg but stops right after an + // skb carrying SCM_RIGHTS (`if (scm.fp) break;`). A client sends a + // head's descriptors with the start of that head, so descriptors + // returned by one receive belong to the last head starting in its + // bytes. Earlier chunks always belong to the current head; the + // last chunk does too unless bytes remain after this head ends. + let mut last = std::mem::take(&mut connection.ahead_descriptors); + if ancillary.is_none() && last.has_any() { + // The caller does not accept descriptors: close and reject. + drop(last); + return Ok(rejected_head(connection, Error::Malformed)); + } if let Some((ahead, range)) = connection.read_ahead.take() { if range.len() > buffer.bytes.len() { return Ok(rejected_head(connection, Error::HeadTooLarge)); @@ -647,6 +736,11 @@ impl HttpIo { buffer.bytes[..end].zeroize(); if used > end { connection.read_ahead = Some((buffer, end..used)); + // A later head starts in the last chunk; keep its + // descriptors for that head. + connection.ahead_descriptors = std::mem::take(&mut last); + } else if let Some(sink) = &ancillary { + sink.borrow_mut().absorb(std::mem::take(&mut last)); } let head = connection.state_mut().admit(head)?; // Admission may remove authentication fields, but it must not @@ -670,15 +764,35 @@ impl HttpIo { buffer = larger; } let end = buffer.bytes.len(); - let completed = self - .reactor - .recv( - connection.socket(), - BufferRange::new::(buffer, used..end)?, - connection, - scope, - ) - .await?; + let range = BufferRange::new::(buffer, used..end)?; + let completed = if let Some(sink) = &ancillary { + let received = self + .reactor + .recv_msg(connection.socket(), range, connection, scope) + .await?; + // The previous chunk did not complete this head, so its + // descriptors belong to it. + let mut sink = sink.borrow_mut(); + sink.absorb(std::mem::take(&mut last)); + last.truncated = received.truncated; + for descriptor in received.descriptors { + if last.descriptors.len() < MAX_HEAD_DESCRIPTORS { + last.descriptors.push(descriptor); + } else { + drop(descriptor); + last.truncated = true; + } + } + Completion { + buffer: received.buffer, + bytes: received.bytes, + lease: received.lease, + } + } else { + self.reactor + .recv(connection.socket(), range, connection, scope) + .await? + }; if completed.bytes == 0 || completed.bytes > end - used { return Err(uring_runtime::Error::Io.into()); } @@ -1044,6 +1158,7 @@ impl ConnectionLease { pool, reusable: false, read_ahead: None, + ahead_descriptors: ReceivedDescriptors::default(), rx_remaining: None, tx_remaining: None, request_is_head: false, @@ -1092,6 +1207,12 @@ impl ConnectionLease { { return Err(Error::Malformed.into()); } + if self.ahead_descriptors.has_any() { + // Carried descriptors without the bytes of their head: close them. + self.ahead_descriptors = ReceivedDescriptors::default(); + self.poison(); + return Err(Error::Malformed.into()); + } self.reusable = false; self.rx_remaining = None; self.tx_remaining = None; @@ -2040,6 +2161,7 @@ fn rejected_head( ) -> HeadCompletion> { connection.poison(); connection.read_ahead = None; + connection.ahead_descriptors = ReceivedDescriptors::default(); connection.rx_remaining = None; connection.request_is_head = false; HeadCompletion { diff --git a/cmd/racer-dataplane/http/tests/exchanges.rs b/cmd/racer-dataplane/http/tests/exchanges.rs index 9c72f253bc..3d1f430e4a 100644 --- a/cmd/racer-dataplane/http/tests/exchanges.rs +++ b/cmd/racer-dataplane/http/tests/exchanges.rs @@ -685,3 +685,213 @@ fn response_on_request_only_connection_can_be_rejected_but_never_reused() { assert!(!response.connection.is_reusable()); assert_eq!(io.reactor().in_flight(), 0); } + +/// Send `bytes` with `fds` as one `SCM_RIGHTS` message on a raw stream. +fn send_with_fds(socket: &UnixStream, bytes: &[u8], fds: &[std::os::fd::RawFd]) { + use std::os::fd::AsRawFd; + let payload = std::mem::size_of_val(fds); + let space = unsafe { libc::CMSG_SPACE(payload as u32) } as usize; + let mut control = vec![0u64; space.div_ceil(8)]; + let mut iov = libc::iovec { + iov_base: bytes.as_ptr().cast_mut().cast(), + iov_len: bytes.len(), + }; + let mut header: libc::msghdr = unsafe { std::mem::zeroed() }; + header.msg_iov = &mut iov; + header.msg_iovlen = 1; + header.msg_control = control.as_mut_ptr().cast(); + header.msg_controllen = space as _; + unsafe { + let cmsg = libc::CMSG_FIRSTHDR(&header); + (*cmsg).cmsg_level = libc::SOL_SOCKET; + (*cmsg).cmsg_type = libc::SCM_RIGHTS; + (*cmsg).cmsg_len = libc::CMSG_LEN(payload as u32) as _; + std::ptr::copy_nonoverlapping(fds.as_ptr().cast::(), libc::CMSG_DATA(cmsg), payload); + } + let sent = unsafe { libc::sendmsg(socket.as_raw_fd(), &header, 0) }; + assert_eq!(sent, bytes.len() as isize); +} + +/// Descriptors sent with any part of a request head are returned with it, and +/// a rejected head closes them. +#[test] +fn request_heads_collect_passed_descriptors() { + use std::io::{Read, Write}; + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + let io = io(); + for malformed in [false, true] { + let (mut client, server) = UnixStream::pair().unwrap(); + let server = ConnectionLease::from_reserved(server.into(), (), ()).unwrap(); + let mut fds = [0; 2]; + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + let mut read = unsafe { std::fs::File::from_raw_fd(fds[0]) }; + let write = unsafe { OwnedFd::from_raw_fd(fds[1]) }; + let first: &[u8] = if malformed { + b"BAD\r\n" + } else { + b"POST /x HTTP/1.1\r\n" + }; + send_with_fds(&client, first, &[write.as_raw_fd(), write.as_raw_fd()]); + drop(write); + client.write_all(b"Content-Length: 0\r\n\r\n").unwrap(); + let received = drive( + &io, + io.receive_request_head_with_descriptors(server, &RequestScope, 128), + ) + .unwrap(); + let mut byte = [0]; + if malformed { + assert!(received.value.is_err()); + } else { + let head = received.value.unwrap(); + assert!(matches!(head.head.start, StartLine::Request { .. })); + assert!(!head.descriptors.truncated); + assert_eq!(head.descriptors.descriptors.len(), 2); + drop(head); + } + // Every copy is closed: the pipe reports EOF. + assert_eq!(read.read(&mut byte).unwrap(), 0); + } + // A plain head reports no descriptors. + let (mut client, server) = UnixStream::pair().unwrap(); + let server = ConnectionLease::from_reserved(server.into(), (), ()).unwrap(); + client + .write_all(b"POST /x HTTP/1.1\r\nContent-Length: 0\r\n\r\n") + .unwrap(); + let head = drive( + &io, + io.receive_request_head_with_descriptors(server, &RequestScope, 128), + ) + .unwrap() + .value + .unwrap(); + assert!(head.descriptors.descriptors.is_empty()); + assert!(!head.descriptors.truncated); +} + +/// Descriptors retained across all fragments of one head are capped; excess +/// descriptors are closed immediately and the head is marked truncated. +#[test] +fn request_head_descriptors_are_capped_across_fragments() { + use std::io::{Read, Write}; + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + let io = io(); + let (mut client, server) = UnixStream::pair().unwrap(); + let server = ConnectionLease::from_reserved(server.into(), (), ()).unwrap(); + let mut fds = [0; 2]; + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + let mut read = unsafe { std::fs::File::from_raw_fd(fds[0]) }; + let write = unsafe { OwnedFd::from_raw_fd(fds[1]) }; + let fragments = http1::connection::MAX_HEAD_DESCRIPTORS + 4; + send_with_fds(&client, b"POST /x HTTP/1.1\r\n", &[write.as_raw_fd()]); + for _ in 1..fragments { + send_with_fds( + &client, + b"X: y\r\n", + &[write.as_raw_fd(), write.as_raw_fd()], + ); + } + drop(write); + client.write_all(b"Content-Length: 0\r\n\r\n").unwrap(); + let head = drive( + &io, + io.receive_request_head_with_descriptors(server, &RequestScope, 4096), + ) + .unwrap() + .value + .unwrap(); + assert!(head.descriptors.truncated); + assert_eq!( + head.descriptors.descriptors.len(), + http1::connection::MAX_HEAD_DESCRIPTORS + ); + drop(head); + let mut byte = [0]; + assert_eq!(read.read(&mut byte).unwrap(), 0); +} + +/// Open a pipe; return its read end and an owned write end. +fn descriptor_probe() -> (std::fs::File, std::os::fd::OwnedFd) { + use std::os::fd::FromRawFd; + let mut fds = [0; 2]; + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + unsafe { + ( + std::fs::File::from_raw_fd(fds[0]), + std::os::fd::OwnedFd::from_raw_fd(fds[1]), + ) + } +} + +/// Receive two pipelined heads; the descriptor goes only with the second. +fn pipelined_heads(second: &[&[u8]]) { + use std::io::{Read, Write}; + use std::os::fd::AsRawFd; + let io = io(); + let (mut client, server) = UnixStream::pair().unwrap(); + let server = ConnectionLease::from_reserved(server.into(), (), ()).unwrap(); + let (mut read, write) = descriptor_probe(); + client.write_all(b"HEAD /a HTTP/1.1\r\n\r\n").unwrap(); + send_with_fds(&client, second[0], &[write.as_raw_fd()]); + drop(write); + for part in &second[1..] { + client.write_all(part).unwrap(); + } + let first = drive( + &io, + io.receive_request_head_with_descriptors(server, &RequestScope, 4096), + ) + .unwrap(); + let head = first.value.unwrap(); + assert!(head.descriptors.descriptors.is_empty()); + assert!(!head.descriptors.truncated); + let connection = first.connection; + let second = drive( + &io, + io.receive_request_head_with_descriptors(connection, &RequestScope, 4096), + ) + .unwrap(); + let head = second.value.unwrap(); + assert!(matches!(head.head.start, StartLine::Request { .. })); + assert_eq!(head.descriptors.descriptors.len(), 1); + assert!(!head.descriptors.truncated); + drop(head); + drop(second.connection); + let mut byte = [0]; + assert_eq!(read.read(&mut byte).unwrap(), 0); +} + +/// A plain head and a descriptor-bearing head read by one `recvmsg`. +#[test] +fn pipelined_head_descriptors_attach_to_their_own_head() { + pipelined_heads(&[b"POST /b HTTP/1.1\r\nContent-Length: 0\r\n\r\n"]); +} + +/// The descriptor-bearing skb carries only part of the second head. +#[test] +fn pipelined_fragmented_head_keeps_its_descriptors() { + pipelined_heads(&[b"POST /b HTTP/1.1\r\n", b"Content-Length: 0\r\n\r\n"]); +} + +/// Descriptors carried with read-ahead close when the connection drops. +#[test] +fn carried_head_descriptors_close_with_the_connection() { + use std::io::{Read, Write}; + use std::os::fd::AsRawFd; + let io = io(); + let (mut client, server) = UnixStream::pair().unwrap(); + let server = ConnectionLease::from_reserved(server.into(), (), ()).unwrap(); + let (mut read, write) = descriptor_probe(); + client.write_all(b"HEAD /a HTTP/1.1\r\n\r\n").unwrap(); + send_with_fds(&client, b"POST /b HTTP/1.1\r\n", &[write.as_raw_fd()]); + drop(write); + let first = drive( + &io, + io.receive_request_head_with_descriptors(server, &RequestScope, 4096), + ) + .unwrap(); + assert!(first.value.unwrap().descriptors.descriptors.is_empty()); + drop(first.connection); + let mut byte = [0]; + assert_eq!(read.read(&mut byte).unwrap(), 0); +} diff --git a/cmd/racer-dataplane/object-wire/src/client.rs b/cmd/racer-dataplane/object-wire/src/client.rs index 81fd1abe10..fd060280af 100644 --- a/cmd/racer-dataplane/object-wire/src/client.rs +++ b/cmd/racer-dataplane/object-wire/src/client.rs @@ -2,7 +2,7 @@ use crate::model::{ ByteRange, CacheKey, MAX_FIELD_BYTES, MAX_WIRE_INTEGER, ObjectMetadata, PAGE_BYTES, - ResolvedRange, StrongEtag, + ResolvedRange, StrongEtag, parse_decimal, }; use crate::{Error, Result}; use http1::{Header, MessageHead, StartLine, is_token, trim_ows}; @@ -28,6 +28,58 @@ pub enum ReadKind { byte_credits: u64, }, + /// Write the plaintext into a client file instead of the socket. + HostRead { + pin: Option, + + range: Option, + + destination: Destination, + }, + /// Write the plaintext into a client GPU allocation. + GpuRead { + pin: Option, + + range: Option, + + destination: Destination, + + /// The size the client passed to `cuMemCreate`. + allocation_size: u64, + }, +} + +/// The client window `[offset, offset + length)` that a direct read may write. +/// The parser guarantees `length > 0` and that the end fits in a wire integer. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Destination { + pub offset: u64, + + pub length: u64, +} + +impl Destination { + /// Exclusive end of the window. + pub fn end(&self) -> u64 { + self.offset + self.length + } +} + +/// Where a direct read puts the plaintext, sent as `Racer-Delivery`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DirectDelivery { + Host, + Gpu, +} + +impl DirectDelivery { + /// The `Racer-Delivery` header value. + pub fn as_str(self) -> &'static str { + match self { + Self::Host => "host", + Self::Gpu => "gpu", + } + } } impl ReadKind { @@ -40,7 +92,39 @@ impl ReadKind { pub fn pin(&self) -> Option<&StrongEtag> { match self { Self::HeadPinned { etag } => Some(etag), - Self::Subscription { pin, .. } => pin.as_ref(), + Self::Subscription { pin, .. } + | Self::HostRead { pin, .. } + | Self::GpuRead { pin, .. } => pin.as_ref(), + _ => None, + } + } + + /// The requested range of a body read, if any. + pub fn range(&self) -> Option { + match self { + Self::Subscription { range, .. } + | Self::HostRead { range, .. } + | Self::GpuRead { range, .. } => *range, + _ => None, + } + } + + /// Whether this request writes into a client descriptor, not the socket. + /// Exactly these requests must carry one descriptor. + pub fn direct(&self) -> Option { + match self { + Self::HostRead { .. } => Some(DirectDelivery::Host), + Self::GpuRead { .. } => Some(DirectDelivery::Gpu), + _ => None, + } + } + + /// The destination window of a direct read. + pub fn destination(&self) -> Option { + match self { + Self::HostRead { destination, .. } | Self::GpuRead { destination, .. } => { + Some(*destination) + } _ => None, } } @@ -86,6 +170,19 @@ impl RequestParser { return Err(Error::InvalidRequest); }; let mut decoded_bytes = method.len().saturating_add(target.len()); + let path = target + .strip_prefix("/v1/objects/") + .ok_or(Error::InvalidRequest)?; + // Only the exact suffixes select a direct read; anything else keeps the + // plain object path and its existing key validation. + let (key_text, direct) = match path.split_once('/') { + Some((key, "host-read")) => (key, Some(DirectDelivery::Host)), + Some((key, "gpu-read")) => (key, Some(DirectDelivery::Gpu)), + _ => (path, None), + }; + let mut destination_offset = None; + let mut destination_length = None; + let mut allocation_size = None; let mut seen = HashSet::new(); let mut host = None; let mut pin = None; @@ -165,8 +262,19 @@ impl RequestParser { pin = Some(StrongEtag::parse(value)?); } "range" => range = Some(ByteRange::parse(value)?), + // Direct reads are paced by the dataplane and ignore credits. + "racer-page-credits" | "racer-byte-credits" if direct.is_some() => {} "racer-page-credits" => page_credits = decimal(value, 1, 64)? as usize, "racer-byte-credits" => byte_credits = decimal(value, PAGE_BYTES, 64 * PAGE_BYTES)?, + "racer-destination-offset" if direct.is_some() => { + once(&mut destination_offset, parse_decimal(value)?)?; + } + "racer-destination-length" if direct.is_some() => { + once(&mut destination_length, parse_decimal(value)?)?; + } + "racer-allocation-size" if direct == Some(DirectDelivery::Gpu) => { + once(&mut allocation_size, parse_decimal(value)?)?; + } "racer-metadata" => { validate_opaque(value)?; metadata = Some(value); @@ -184,26 +292,56 @@ impl RequestParser { if host != Some(true) { return Err(Error::InvalidRequest); } - let key = CacheKey::parse_hex( - target - .strip_prefix("/v1/objects/") - .ok_or(Error::InvalidRequest)? - .as_bytes(), - )?; - let kind = match method.as_str() { - "HEAD" if range.is_none() => match pin { - Some(etag) => ReadKind::HeadPinned { etag }, - None => ReadKind::Head, - }, - "HEAD" => return Err(Error::InvalidRequest), - "POST" if content_length => ReadKind::Subscription { - pin, - range, - page_credits, - byte_credits, + let key = CacheKey::parse_hex(key_text.as_bytes())?; + let kind = match (method.as_str(), direct) { + ("POST", Some(_)) if !content_length => return Err(Error::InvalidRequest), + ("POST", Some(delivery)) => { + let destination = Destination { + offset: destination_offset.ok_or(Error::InvalidRequest)?, + length: destination_length + .filter(|&length| length != 0) + .ok_or(Error::InvalidRequest)?, + }; + destination + .offset + .checked_add(destination.length) + .filter(|&end| end <= MAX_WIRE_INTEGER) + .ok_or(Error::InvalidRequest)?; + match delivery { + DirectDelivery::Host => ReadKind::HostRead { + pin, + range, + destination, + }, + DirectDelivery::Gpu => { + let allocation_size = allocation_size + .filter(|&size| destination.end() <= size) + .ok_or(Error::InvalidRequest)?; + ReadKind::GpuRead { + pin, + range, + destination, + allocation_size, + } + } + } + } + (_, Some(_)) => return Err(Error::MethodNotAllowed), + (method, None) => match method { + "HEAD" if range.is_none() => match pin { + Some(etag) => ReadKind::HeadPinned { etag }, + None => ReadKind::Head, + }, + "HEAD" => return Err(Error::InvalidRequest), + "POST" if content_length => ReadKind::Subscription { + pin, + range, + page_credits, + byte_credits, + }, + "POST" => return Err(Error::InvalidRequest), + _ => return Err(Error::MethodNotAllowed), }, - "POST" => return Err(Error::InvalidRequest), - _ => return Err(Error::MethodNotAllowed), }; Ok(ClientRequest { kind, @@ -271,6 +409,14 @@ fn parse_max_age(value: &[u8], max_age: &mut Option) -> Result<()> { Ok(()) } +/// Store a destination field, rejecting a repeated one. +fn once(slot: &mut Option, value: u64) -> Result<()> { + if slot.replace(value).is_some() { + return Err(Error::InvalidRequest); + } + Ok(()) +} + /// Parse a nonzero canonical credit value inside its protocol bounds. fn decimal(value: &[u8], minimum: u64, maximum: u64) -> Result { if value.is_empty() || value[0] == b'0' || !value.iter().all(u8::is_ascii_digit) { @@ -307,6 +453,15 @@ fn validate_opaque(value: &[u8]) -> Result<()> { pub fn subscription_head( metadata: &ObjectMetadata, range: Option, +) -> Result { + body_head(metadata, range, true) +} + +/// Shared Get-style head. Without payload, page frames carry only their 21 bytes. +fn body_head( + metadata: &ObjectMetadata, + range: Option, + payload: bool, ) -> Result { let mut head = success_head(metadata, None)?; head.headers @@ -323,7 +478,7 @@ pub fn subscription_head( let length = pages .checked_add(1) .and_then(|n| n.checked_mul(21)) - .and_then(|n| n.checked_add(end - start)) + .and_then(|n| n.checked_add(if payload { end - start } else { 0 })) .filter(|&n| n <= MAX_WIRE_INTEGER) .ok_or(Error::BadGateway)?; head.start = StartLine::Response { status: 200 }; @@ -337,6 +492,24 @@ pub fn subscription_head( Ok(head) } +/// Encode a direct-read success head. Page frames carry no payload, so the +/// body is `21 * (pages + 1)` bytes. An empty object has no pages. +pub fn direct_head( + metadata: &ObjectMetadata, + range: Option, + delivery: DirectDelivery, +) -> Result { + let mut head = body_head(metadata, range, false)?; + let close = head + .headers + .iter() + .position(|h| h.name.eq_ignore_ascii_case("Connection")) + .unwrap_or(head.headers.len()); + head.headers + .insert(close, header("Racer-Delivery", delivery.as_str())); + Ok(head) +} + /// Encode the fixed-width page or completion frame without managing page leases. pub fn frame(kind: u8, number: u64, offset: u64, length: u32) -> [u8; 21] { let mut bytes = [0; 21]; diff --git a/cmd/racer-dataplane/object-wire/tests/direct.rs b/cmd/racer-dataplane/object-wire/tests/direct.rs new file mode 100644 index 0000000000..8e2e5fe917 --- /dev/null +++ b/cmd/racer-dataplane/object-wire/tests/direct.rs @@ -0,0 +1,283 @@ +//! Direct-read (`host-read`, `gpu-read`) request parsing and response heads. + +use http1::{Header, MessageHead, StartLine}; +use racer_control_wire::CacheId; +use racer_object_wire::client::{self, Destination, DirectDelivery, ReadKind, RequestParser}; +use racer_object_wire::*; + +fn key() -> String { + CacheKey([0xab; 32]).to_hex() +} + +/// A direct request with valid destination fields plus `extra`. +fn direct(method: &str, suffix: &str, extra: &[(&str, &str)]) -> MessageHead { + let mut headers = vec![ + Header::new("Host", "racer"), + Header::new("Content-Length", "0"), + ]; + headers.extend(extra.iter().map(|(name, value)| Header::new(*name, *value))); + MessageHead { + start: StartLine::Request { + method: method.into(), + target: format!("/v1/objects/{}/{suffix}", key()), + }, + headers, + } +} + +fn parse(head: &MessageHead) -> Result { + RequestParser::new(usize::MAX) + .parse(head) + .map(|request| request.kind) +} + +const DEST: [(&str, &str); 2] = [ + ("Racer-Destination-Offset", "4096"), + ("Racer-Destination-Length", "100"), +]; + +#[test] +fn host_read_parses_range_pin_and_destination() { + let head = direct( + "POST", + "host-read", + &[ + DEST[0], + DEST[1], + ("Range", "bytes=1-3"), + ("If-Match", "\"v1\""), + // Credits are ignored, even when they would be invalid on a Get. + ("Racer-Page-Credits", "0"), + ("Racer-Byte-Credits", "x"), + ("Racer-Allocation-Size", "junk"), + ], + ); + let request = RequestParser::new(usize::MAX).parse(&head).unwrap(); + assert_eq!(request.key, CacheKey([0xab; 32])); + let kind = request.kind; + assert_eq!( + kind, + ReadKind::HostRead { + pin: Some(StrongEtag::parse(b"\"v1\"").unwrap()), + range: Some(ByteRange::closed(1, 3).unwrap()), + destination: Destination { + offset: 4096, + length: 100 + }, + } + ); + assert_eq!(kind.direct(), Some(DirectDelivery::Host)); + assert_eq!(kind.destination().unwrap().end(), 4196); + assert_eq!(kind.range(), Some(ByteRange::closed(1, 3).unwrap())); + assert!(kind.pin().is_some()); + assert!(!kind.is_head()); +} + +#[test] +fn gpu_read_requires_allocation_that_holds_the_window() { + let parsed = parse(&direct( + "POST", + "gpu-read", + &[DEST[0], DEST[1], ("Racer-Allocation-Size", "4196")], + )) + .unwrap(); + assert_eq!( + parsed, + ReadKind::GpuRead { + pin: None, + range: None, + destination: Destination { + offset: 4096, + length: 100 + }, + allocation_size: 4196, + } + ); + assert_eq!(parsed.direct(), Some(DirectDelivery::Gpu)); + for extra in [ + &[DEST[0], DEST[1]][..], + &[DEST[0], DEST[1], ("Racer-Allocation-Size", "4195")], + &[DEST[0], DEST[1], ("Racer-Allocation-Size", "-1")], + &[ + DEST[0], + DEST[1], + ("Racer-Allocation-Size", "9999"), + ("Racer-Allocation-Size", "9999"), + ], + ] { + assert_eq!( + parse(&direct("POST", "gpu-read", extra)), + Err(Error::InvalidRequest), + "{extra:?}" + ); + } +} + +#[test] +fn destination_fields_are_required_canonical_unique_and_bounded() { + let max = i64::MAX.to_string(); + for suffix in ["host-read", "gpu-read"] { + let size = ("Racer-Allocation-Size", max.as_str()); + for extra in [ + &[size][..], + &[size, DEST[0]], + &[size, DEST[1]], + &[size, DEST[0], ("Racer-Destination-Length", "0")], + &[size, DEST[0], ("Racer-Destination-Length", "01")], + &[size, DEST[1], ("Racer-Destination-Offset", " 1")], + &[ + size, + DEST[1], + ("Racer-Destination-Offset", "9223372036854775808"), + ], + &[size, DEST[0], DEST[1], DEST[1]], + &[size, DEST[0], DEST[0], DEST[1]], + // offset + length must stay inside a wire integer. + &[ + size, + ("Racer-Destination-Offset", max.as_str()), + ("Racer-Destination-Length", "1"), + ], + ] { + assert_eq!( + parse(&direct("POST", suffix, extra)), + Err(Error::InvalidRequest), + "{suffix} {extra:?}" + ); + } + let ok = parse(&direct( + "POST", + suffix, + &[ + size, + ("Racer-Destination-Offset", "0"), + ("Racer-Destination-Length", max.as_str()), + ], + )) + .unwrap(); + assert_eq!(ok.destination().unwrap().end(), i64::MAX as u64); + } +} + +#[test] +fn direct_paths_require_post_with_zero_length_and_known_suffix() { + for method in ["GET", "HEAD", "PUT"] { + assert_eq!( + parse(&direct(method, "host-read", &DEST)), + Err(Error::MethodNotAllowed) + ); + } + let mut head = direct("POST", "host-read", &DEST); + head.headers.retain(|h| h.name != "Content-Length"); + assert_eq!(parse(&head), Err(Error::InvalidRequest)); + for suffix in ["cpu-read", "host-read/", "", "HOST-READ"] { + assert_eq!( + parse(&direct("POST", suffix, &DEST)), + Err(Error::InvalidRequest), + "{suffix}" + ); + } + // The key is still validated on direct paths. + let mut head = direct("POST", "host-read", &DEST); + head.start = StartLine::Request { + method: "POST".into(), + target: format!("/v1/objects/{}/host-read", "A".repeat(64)), + }; + assert_eq!(parse(&head), Err(Error::InvalidRequest)); +} + +#[test] +fn plain_requests_ignore_destination_fields_as_before() { + let mut head = direct("POST", "", &[]); + head.start = StartLine::Request { + method: "POST".into(), + target: format!("/v1/objects/{}", key()), + }; + let plain = parse(&head).unwrap(); + head.headers.extend([ + Header::new("Racer-Destination-Offset", "x"), + Header::new("Racer-Destination-Offset", "y"), + Header::new("Racer-Allocation-Size", "z"), + ]); + assert_eq!(parse(&head).unwrap(), plain); + assert!(matches!(plain, ReadKind::Subscription { .. })); + assert_eq!(plain.direct(), None); + assert_eq!(plain.destination(), None); +} + +fn metadata(length: u64) -> ObjectMetadata { + ObjectMetadata { + content_type: None, + version: ObjectVersion { + object: ObjectId { + cache: CacheId("cache".into()), + key: CacheKey([0xab; 32]), + }, + etag: StrongEtag::parse(b"\"v1\"").unwrap(), + }, + length, + expires_at: ExpiresAt::from_unix_millis(1234).unwrap(), + } +} + +#[test] +fn direct_head_counts_only_frames_and_names_delivery() { + let length = 2 * PAGE_BYTES + 5; + for (range, delivery, pages) in [ + (Some((0, length)), DirectDelivery::Host, 3), + ( + Some((PAGE_BYTES - 1, PAGE_BYTES + 1)), + DirectDelivery::Gpu, + 2, + ), + (Some((3, 4)), DirectDelivery::Host, 1), + ] { + let range = range.map(|(start, end)| { + ByteRange::closed(start, end - 1) + .unwrap() + .resolve(length) + .unwrap() + }); + let head = client::direct_head(&metadata(length), range, delivery).unwrap(); + let subscription = client::subscription_head(&metadata(length), range).unwrap(); + assert!(matches!(head.start, StartLine::Response { status: 200 })); + assert_eq!( + head.unique("Content-Length").unwrap(), + Some((21 * (pages + 1)).to_string().as_bytes()) + ); + assert_eq!( + head.unique("Racer-Delivery").unwrap(), + Some(delivery.as_str().as_bytes()) + ); + assert_eq!( + head.unique("Connection").unwrap(), + Some(b"close".as_slice()) + ); + // Every other field matches the Get head. + for field in &subscription.headers { + if !field.name.eq_ignore_ascii_case("Content-Length") { + assert_eq!( + head.unique(&field.name).unwrap(), + Some(field.value.as_slice()) + ); + } + } + assert_eq!(head.headers.len(), subscription.headers.len() + 1); + } + let empty = client::direct_head(&metadata(0), None, DirectDelivery::Host).unwrap(); + assert_eq!( + empty.unique("Content-Length").unwrap(), + Some(b"21".as_slice()) + ); + // A maximum-length object fits because frames carry no payload. + let range = ByteRange::From(0).resolve(MAX_WIRE_INTEGER).unwrap(); + assert!( + client::direct_head( + &metadata(MAX_WIRE_INTEGER), + Some(range), + DirectDelivery::Host + ) + .is_ok() + ); + assert!(client::subscription_head(&metadata(MAX_WIRE_INTEGER), Some(range)).is_err()); +} diff --git a/cmd/racer-dataplane/runtime/src/reactor.rs b/cmd/racer-dataplane/runtime/src/reactor.rs index 11499201f0..af45cb84e7 100644 --- a/cmd/racer-dataplane/runtime/src/reactor.rs +++ b/cmd/racer-dataplane/runtime/src/reactor.rs @@ -177,6 +177,140 @@ pub struct Completion { pub lease: L, } +/// Most `SCM_RIGHTS` descriptors one [`Reactor::recv_msg`] accepts. More set +/// [`MessageCompletion::truncated`], and the kernel closes the excess. +pub const MAX_RECEIVED_DESCRIPTORS: usize = 8; + +/// A fenced `recvmsg` result with any descriptors that arrived with the bytes. +pub struct MessageCompletion { + /// Original allocation, returned only after all applicable CQEs. + pub buffer: B, + + /// Bytes received, possibly less than the buffer length. + pub bytes: usize, + + /// Caller-owned admission or reuse guard retained through completion. + pub lease: L, + + /// Received `SCM_RIGHTS` descriptors, already close-on-exec. + pub descriptors: Vec, + + /// The kernel reported `MSG_CTRUNC`: ancillary data did not fit. + pub truncated: bool, +} + +/// Heap-pinned `msghdr`, `iovec`, and control storage for one `RecvMsg`. +/// Any descriptors the kernel installed are closed when this is dropped. +struct MessageHeader { + header: libc::msghdr, + + iov: libc::iovec, + + control: Vec, +} + +impl MessageHeader { + /// Control bytes able to hold [`MAX_RECEIVED_DESCRIPTORS`] descriptors. + fn control_bytes() -> usize { + // SAFETY: CMSG_SPACE only computes an aligned size. + unsafe { + libc::CMSG_SPACE((MAX_RECEIVED_DESCRIPTORS * std::mem::size_of::()) as u32) + as usize + } + } + + fn new() -> Box { + let words = Self::control_bytes().div_ceil(std::mem::size_of::()); + Box::new(Self { + // SAFETY: msghdr is plain old data; all-zero is a valid empty value. + header: unsafe { std::mem::zeroed() }, + iov: libc::iovec { + iov_base: std::ptr::null_mut(), + iov_len: 0, + }, + control: vec![0; words], + }) + } + + /// Point the header at `ptr..ptr+len` and an empty control buffer. + fn prepare(&mut self, ptr: *mut u8, len: u32) -> *mut libc::msghdr { + // Close anything left by an earlier attempt before reusing the storage. + drop(self.take_descriptors()); + self.control.fill(0); + self.iov = libc::iovec { + iov_base: ptr.cast(), + iov_len: len as usize, + }; + // SAFETY: as in new(). + self.header = unsafe { std::mem::zeroed() }; + self.header.msg_iov = &mut self.iov; + self.header.msg_iovlen = 1; + self.header.msg_control = self.control.as_mut_ptr().cast(); + self.header.msg_controllen = Self::control_bytes() as _; + &mut self.header + } + + /// Whether the kernel truncated ancillary data on the completed receive. + fn truncated(&self) -> bool { + self.header.msg_flags & libc::MSG_CTRUNC != 0 + } + + /// Take ownership of every `SCM_RIGHTS` descriptor in the control buffer. + /// The buffer is cleared, so a second call or the drop finds nothing. + // Control lengths are usize on glibc but narrower on other libcs. + #[allow(clippy::unnecessary_cast)] + fn take_descriptors(&mut self) -> Vec { + let mut descriptors = Vec::new(); + let capacity = self.control.len() * std::mem::size_of::(); + let used = (self.header.msg_controllen as usize).min(capacity); + if self.header.msg_control.is_null() || used < std::mem::size_of::() { + return descriptors; + } + let base = self.control.as_ptr().cast::(); + let mut offset = 0; + while offset + std::mem::size_of::() <= used { + // SAFETY: the header lies inside the initialized control buffer; + // read_unaligned tolerates any alignment. + let cmsg: libc::cmsghdr = unsafe { std::ptr::read_unaligned(base.add(offset).cast()) }; + let length = cmsg.cmsg_len as usize; + if length < std::mem::size_of::() || offset + length > used { + break; + } + // SAFETY: CMSG_LEN(0) is the aligned header size. + let data = unsafe { libc::CMSG_LEN(0) } as usize; + if cmsg.cmsg_level == libc::SOL_SOCKET && cmsg.cmsg_type == libc::SCM_RIGHTS { + let count = (length.saturating_sub(data)) / std::mem::size_of::(); + for index in 0..count { + // SAFETY: the descriptor array is inside this control message. + let fd: i32 = unsafe { + std::ptr::read_unaligned( + base.add(offset + data + index * std::mem::size_of::()) + .cast(), + ) + }; + if fd >= 0 { + // SAFETY: the kernel installed this descriptor for us alone. + descriptors.push(unsafe { HostFd::from_raw_fd(fd) }); + } + } + } + // CMSG_NXTHDR advances by CMSG_ALIGN(cmsg_len). + let step = length.next_multiple_of(std::mem::size_of::()); + offset += step; + } + self.control.fill(0); + self.header.msg_controllen = 0; + descriptors + } +} + +impl Drop for MessageHeader { + /// Close descriptors that no caller took, including after an abandoned wait. + fn drop(&mut self) { + drop(self.take_descriptors()); + } +} + /// An exclusively owned, stable backing allocation with an independent lifetime. /// /// # Safety @@ -1720,6 +1854,132 @@ impl Reactor { }) } + /// Receive like [`Self::recv`], also accepting `SCM_RIGHTS` descriptors with + /// `MSG_CMSG_CLOEXEC`. Descriptors are owned before this returns; a dropped + /// or failed wait closes any the kernel delivered. Simulated sockets carry + /// no ancillary data and always report none. + pub fn recv_msg<'a, B: IoBuffer, L: 'static>( + &'a self, + fd: Rc, + buffer: B, + lease: L, + scope: &'a S, + ) -> Operation<'a, MessageCompletion, S::Error> + where + S::Error: From, + { + Box::pin(async move { + let retained_fd = fd.clone(); + let ((owned, mut header), bytes) = self + .retry_data( + ( + InFlight { + file: fd, + buffer, + lease, + }, + MessageHeader::new(), + ), + retained_fd, + scope, + None, + libc::POLLIN as u32, + |(owned, header)| { + let (ptr, len) = + Self::buffer_region(&mut owned.buffer, BufferOperation::Recv)?; + let sqe = submission!( + self, + simulation::Op::Buffer { + fd: owned.file.clone(), + operation: BufferOperation::Recv, + ptr, + len: len as usize, + }, + opcode::RecvMsg::new( + types::Fd(owned.file.as_raw_fd()), + header.prepare(ptr, len) + ) + .flags(libc::MSG_CMSG_CLOEXEC as u32) + .build() + ); + Ok(sqe) + }, + ) + .await?; + let truncated = header.truncated(); + let descriptors = header.take_descriptors(); + let Completion { + buffer, + bytes, + lease, + } = owned.complete(bytes); + Ok(MessageCompletion { + buffer, + bytes, + lease, + descriptors, + truncated, + }) + }) + } + + /// Write immutable shared storage at an explicit file offset, returning + /// short progress without replaying bytes. Unlike [`Self::write_at`], the + /// buffer needs no mutable capability, so verified pages can be written. + pub fn write_shared_at<'a, B: SendBuffer, L: 'static>( + &'a self, + file: Rc, + offset: u64, + buffer: B, + lease: L, + scope: &'a S, + ) -> Operation<'a, Completion, S::Error> + where + S::Error: From, + { + Box::pin(async move { + if offset > i64::MAX as u64 { + return Err(Error::InvalidInput.into()); + } + let retained_fd = file.clone(); + let (owned, bytes) = self + .retry_data( + InFlight { + file, + buffer, + lease, + }, + retained_fd, + scope, + None, + libc::POLLOUT as u32, + |owned| { + let bytes = owned.buffer.send_bytes()?; + let len = u32::try_from(bytes.len()).map_err(|_| Error::InvalidInput)?; + let sqe = submission!( + self, + simulation::Op::Buffer { + fd: owned.file.clone(), + operation: BufferOperation::Write(offset), + ptr: bytes.as_ptr().cast_mut(), + len: len as usize, + }, + opcode::Write::new( + types::Fd(owned.file.as_raw_fd()), + bytes.as_ptr(), + len + ) + .offset(offset) + .build() + ); + Ok(sqe) + }, + ) + .await?; + Ok(owned.complete(bytes)) + }) + } + /// Select a readiness opcode without converting simulated owners to host FDs. fn readiness_submission(&self, fd: &Rc, interest: u32) -> Submission { submission!( diff --git a/cmd/racer-dataplane/runtime/src/reactor/tests.rs b/cmd/racer-dataplane/runtime/src/reactor/tests.rs index 3657dfcffa..2f67cc3630 100644 --- a/cmd/racer-dataplane/runtime/src/reactor/tests.rs +++ b/cmd/racer-dataplane/runtime/src/reactor/tests.rs @@ -3928,3 +3928,200 @@ mod table { assert!(table.slots.is_empty()); } } + +mod message { + //! `RecvMsg` descriptor passing and positioned shared-buffer writes. + use super::*; + use std::io::{Read, Write}; + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; + + /// Send `bytes` with `fds` as one `SCM_RIGHTS` message. + fn send_with_fds(socket: &UnixStream, bytes: &[u8], fds: &[RawFd]) { + let payload = std::mem::size_of_val(fds); + let space = unsafe { libc::CMSG_SPACE(payload as u32) } as usize; + let mut control = vec![0u64; space.div_ceil(8)]; + let mut iov = libc::iovec { + iov_base: bytes.as_ptr().cast_mut().cast(), + iov_len: bytes.len(), + }; + let mut header: libc::msghdr = unsafe { std::mem::zeroed() }; + header.msg_iov = &mut iov; + header.msg_iovlen = 1; + if !fds.is_empty() { + header.msg_control = control.as_mut_ptr().cast(); + header.msg_controllen = space as _; + unsafe { + let cmsg = libc::CMSG_FIRSTHDR(&header); + (*cmsg).cmsg_level = libc::SOL_SOCKET; + (*cmsg).cmsg_type = libc::SCM_RIGHTS; + (*cmsg).cmsg_len = libc::CMSG_LEN(payload as u32) as _; + std::ptr::copy_nonoverlapping( + fds.as_ptr().cast::(), + libc::CMSG_DATA(cmsg), + payload, + ); + } + } + let sent = unsafe { libc::sendmsg(socket.as_raw_fd(), &header, 0) }; + assert_eq!( + sent, + bytes.len() as isize, + "{}", + std::io::Error::last_os_error() + ); + } + + fn pipe() -> (std::fs::File, OwnedFd) { + let mut fds = [0; 2]; + assert_eq!(unsafe { libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC) }, 0); + unsafe { + ( + std::fs::File::from_raw_fd(fds[0]), + OwnedFd::from_raw_fd(fds[1]), + ) + } + } + + fn memfd() -> std::fs::File { + let fd = unsafe { libc::memfd_create(c"racer-test".as_ptr(), libc::MFD_CLOEXEC) }; + assert!(fd >= 0); + unsafe { std::fs::File::from_raw_fd(fd) } + } + + fn cloexec(fd: &OwnedFd) -> bool { + unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFD) & libc::FD_CLOEXEC != 0 } + } + + #[test] + /// One descriptor arrives owned and close-on-exec with its bytes. + fn receives_one_descriptor_with_bytes() { + let Some(reactor) = kernel_reactor(4) else { + return; + }; + let request = scope(); + let (socket, peer) = UnixStream::pair().unwrap(); + socket.set_nonblocking(true).unwrap(); + let mut file = memfd(); + send_with_fds(&peer, b"head", &[file.as_raw_fd()]); + let drops = Rc::new(Cell::new(0)); + let completed = drive( + &reactor, + reactor.recv_msg( + Rc::new(Descriptor::from(socket)), + Buffer(vec![0; 16], drops.clone()), + Lease(drops.clone()), + &request, + ), + ) + .unwrap(); + assert_eq!(completed.bytes, 4); + assert_eq!(&completed.buffer.0[..4], b"head"); + assert!(!completed.truncated); + assert_eq!(completed.descriptors.len(), 1); + assert!(cloexec(&completed.descriptors[0])); + let mut received = std::fs::File::from(completed.descriptors.into_iter().next().unwrap()); + received.write_all(b"shared").unwrap(); + let mut output = String::new(); + use std::io::Seek; + file.seek(std::io::SeekFrom::Start(0)).unwrap(); + file.read_to_string(&mut output).unwrap(); + assert_eq!(output, "shared"); + } + + #[test] + /// Plain bytes report no descriptors; too many descriptors set truncation. + fn plain_bytes_and_truncated_control_data() { + let Some(reactor) = kernel_reactor(4) else { + return; + }; + let request = scope(); + let (socket, peer) = UnixStream::pair().unwrap(); + socket.set_nonblocking(true).unwrap(); + let socket = Rc::new(Descriptor::from(socket)); + (&peer).write_all(b"x").unwrap(); + let completed = drive( + &reactor, + reactor.recv_msg(socket.clone(), buffer(&[0; 4]), (), &request), + ) + .unwrap(); + assert_eq!(completed.bytes, 1); + assert!(completed.descriptors.is_empty()); + assert!(!completed.truncated); + + let (mut read, write) = pipe(); + let many = vec![write.as_raw_fd(); MAX_RECEIVED_DESCRIPTORS + 1]; + send_with_fds(&peer, b"y", &many); + drop(write); + let completed = drive( + &reactor, + reactor.recv_msg(socket, buffer(&[0; 4]), (), &request), + ) + .unwrap(); + assert_eq!(completed.bytes, 1); + assert!(completed.truncated); + assert!(completed.descriptors.len() <= MAX_RECEIVED_DESCRIPTORS); + drop(completed); + // Every copy of the write end is closed: no descriptor leaked. + let mut byte = [0]; + assert_eq!(read.read(&mut byte).unwrap(), 0); + } + + #[test] + /// A wait dropped before its CQE still closes the delivered descriptor. + fn abandoned_receive_closes_descriptors() { + let Some(reactor) = kernel_reactor(4) else { + return; + }; + let request = scope(); + let (socket, peer) = UnixStream::pair().unwrap(); + socket.set_nonblocking(true).unwrap(); + let drops = Rc::new(Cell::new(0)); + let mut receive = reactor.recv_msg( + Rc::new(Descriptor::from(socket)), + Buffer(vec![0; 4], drops.clone()), + Lease(drops.clone()), + &request, + ); + assert!(poll(&mut receive).is_pending()); + reactor.wait(Duration::ZERO).unwrap(); + drop(receive); + let (mut read, write) = pipe(); + send_with_fds(&peer, b"z", &[write.as_raw_fd()]); + drop(write); + let deadline = Instant::now() + Duration::from_secs(5); + while drops.get() != 2 { + assert!(Instant::now() < deadline, "abandoned receive never fenced"); + reactor.poll_budgeted(8).unwrap(); + reactor.wait(Duration::from_millis(1)).unwrap(); + } + let mut byte = [0]; + assert_eq!(read.read(&mut byte).unwrap(), 0); + } + + #[test] + /// Positioned shared writes land at their offset and reject huge offsets. + fn shared_write_at_offset() { + let Some(reactor) = kernel_reactor(4) else { + return; + }; + let request = scope(); + let mut file = memfd(); + let fd = Rc::new(Descriptor::from(OwnedFd::from(file.try_clone().unwrap()))); + let completed = drive( + &reactor, + reactor.write_shared_at(fd.clone(), 3, buffer(b"abc"), (), &request), + ) + .unwrap(); + assert_eq!(completed.bytes, 3); + let mut output = Vec::new(); + file.read_to_end(&mut output).unwrap(); + assert_eq!(output, b"\0\0\0abc"); + assert!(matches!( + drive( + &reactor, + reactor.write_shared_at(fd, i64::MAX as u64 + 1, buffer(b"a"), (), &request), + ), + Err(Error::InvalidInput) + )); + } +} diff --git a/cmd/racer-dataplane/src/app.rs b/cmd/racer-dataplane/src/app.rs index c8752a3812..591b46b1be 100644 --- a/cmd/racer-dataplane/src/app.rs +++ b/cmd/racer-dataplane/src/app.rs @@ -131,6 +131,8 @@ pub struct Application { /// graph crosses a thread. Worker zero alone drives enrollment/control reloads. pub struct NodeState { devices: Option, + /// The process-wide GPU helper, when `gpu-read` is enabled. + gpu: Option>, inventory: Arc, /// Startup-fixed selector and claimed disks for `/debug/hardware`. @@ -209,6 +211,7 @@ impl NodeState { Ok(Self { peer_admission, devices: None, + gpu: None, peer_receive: Default::default(), inventory: Arc::new(crate::rdma::Inventory::default()), hardware: Default::default(), @@ -283,6 +286,19 @@ impl Application { self.config.limits.retained_snapshots.get(), ))); self.node.inventory.update(self.discovered_nics.clone())?; + Arc::get_mut(&mut self.node).ok_or(Error::Internal)?.gpu = + crate::gpu::GpuHelper::start(self.config.enable_gpu, self.config.gpu_staging_bytes)?; + // GPU decrypt and RDMA together: peers' NICs may write gpu-read pages + // straight into GPU landing buffers (GPUDirect RDMA). Activation + // registers them per rail and leaves a rail host-only on any failure. + if self.config.enable_rdma + && let Some(gpu) = &self.node.gpu + { + self.node + .native + .link_node() + .set_gpu_landing(gpu.clone(), self.config.gpu_landing_bytes); + } if self.config.enable_rdma { self.node.native.place(&plan)?; self.node @@ -1196,10 +1212,12 @@ impl WorkerApplication { runtime.reactor.clone(), admission.clone(), )); - let responses = Rc::new( - Responses::new(io.clone(), delivery) - .with_observer(admission.policy().observer().clone()), - ); + let mut responses = Responses::new(io.clone(), delivery) + .with_observer(admission.policy().observer().clone()); + if let Some(gpu) = &node.gpu { + responses = responses.with_gpu(Rc::new(crate::gpu::GpuClient::new(gpu.clone()))); + } + let responses = Rc::new(responses); let clients = ClientListeners::new( coordinator, RequestParser::new(config.limits.header_bytes.get()), diff --git a/cmd/racer-dataplane/src/client.rs b/cmd/racer-dataplane/src/client.rs index c0dcacc404..c9d0520679 100644 --- a/cmd/racer-dataplane/src/client.rs +++ b/cmd/racer-dataplane/src/client.rs @@ -23,6 +23,7 @@ use crate::model::PageNumber; use crate::model::ResolvedRange; use crate::read::ReadPayload; use crate::read::ReadResponse; +use crate::read::range_stream::DirectItem; use crate::read::range_stream::RangeStream; use crate::runtime::RequestScope; use crate::security::Authorization; @@ -41,9 +42,15 @@ use std::task::Poll; use std::time::Duration; use uring_runtime::reactor::IoBuffer; +pub mod direct; pub mod listener; pub mod origin_directory; +pub use direct::DIRECT_WINDOW_PAGES; +use direct::DirectPage; +use direct::DirectSink; +use direct::GpuService; + use racer_object_wire::client::MAX_HEAD_BYTES; pub use racer_object_wire::client::ReadKind; use racer_object_wire::client::frame; @@ -115,6 +122,7 @@ async fn handle_read_result( scope: &RequestScope, observation: &mut crate::telemetry::RequestMetrics, timeout: Duration, + sink: Option>, ) -> Result> { let response = match read_result { Ok(response) => response, @@ -162,7 +170,19 @@ async fn handle_read_result( > 4; let mut watch = watch_room.then(|| responses.io.disconnected(&connection, &watch_scope)); let mut poll_disconnect = !watch_room; - let mut send = if matches!(kind, ReadKind::Subscription { .. }) { + let mut send = if let Some(sink) = sink { + responses.send_direct( + connection, + response, + kind, + sink, + scope, + observation, + timeout, + ) + } else if kind.direct().is_some() { + Box::pin(async { Err(Error::Internal) }) + } else if matches!(kind, ReadKind::Subscription { .. }) { responses.send_subscription(connection, response, scope, observation, timeout) } else { responses.send_observed(connection, response, scope, observation) @@ -225,6 +245,8 @@ pub struct Responses { io: Rc, delivery: Rc, + + gpu: Option>, } /// Duplex subscription framing. Delivered leases stay pinned until exact release. @@ -619,15 +641,229 @@ impl Responses { }) } + /// Deliver a direct read: write each page through `sink`, then send its + /// payload-less page frame. The sink is finished before the kind 2 frame, + /// and on a best-effort basis when delivery fails. + #[allow(clippy::too_many_arguments)] + pub(crate) fn send_direct<'a>( + &'a self, + connection: ConnectionLease, + response: ReadResponse, + kind: &'a ReadKind, + mut sink: Box, + scope: &'a RequestScope, + observation: &'a mut crate::telemetry::RequestMetrics, + timeout: Duration, + ) -> Operation<'a, ConnectionLease> { + Box::pin(async move { + let result = self + .send_direct_inner( + connection, + response, + kind, + sink.as_mut(), + scope, + observation, + timeout, + ) + .await; + if result.is_err() { + // Give device work a short, independent bound to settle. + if let Ok(final_scope) = RequestScope::new( + scope.request, + uring_runtime::environment::now() + Duration::from_secs(1), + ) { + let _ = sink.finish(&final_scope).await; + } + } + result + }) + } + + #[allow(clippy::too_many_arguments)] + async fn send_direct_inner( + &self, + mut connection: ConnectionLease, + mut response: ReadResponse, + kind: &ReadKind, + sink: &mut dyn DirectSink, + scope: &RequestScope, + observation: &mut crate::telemetry::RequestMetrics, + timeout: Duration, + ) -> Result { + let delivery = kind.direct().ok_or(Error::Internal)?; + let destination = kind.destination().ok_or(Error::Internal)?; + let head = racer_object_wire::client::direct_head( + &response.metadata, + response.payload.range(), + delivery, + )?; + let expected = response.payload.range().map_or(0, |r| r.len()); + if expected > destination.length { + return Err(Error::InvalidRequest); + } + let mut first = if let ReadPayload::Stream(stream) = &mut response.payload { + // A GPU sink that decrypts asks for sealed pages, so the CPU + // decrypts only pages it holds as plaintext anyway. + if let Some(target) = sink.sealed_target() { + stream.set_sealed(target)?; + } + match stream.next_direct().await { + Ok(Some(item)) => Some(item), + result => { + let error = result.err().unwrap_or(Error::BadGateway); + self.observer.record( + crate::telemetry::Failure::new(crate::telemetry::Stage::FirstSlice, error) + .request(scope), + ); + observation.fail(error); + return self.send_error(connection, error, scope).await; + } + } + } else { + None + }; + connection.poison(); + connection = self.io.send_head(connection, head, scope).await?.connection; + let mut sent = 0; + let mut pages = 0; + if let ReadPayload::Stream(stream) = &mut response.payload { + let range = stream.range(); + stream.enable_progress(timeout); + while sent < expected { + let mut progress_scope = scope.clone(); + progress_scope.deadline.0 = uring_runtime::environment::now() + timeout; + let item = match first.take() { + Some(item) => item, + None => std::future::poll_fn(|cx| { + progress_scope.check()?; + stream.next_direct().as_mut().poll(cx) + }) + .await + .inspect_err(|&error| { + self.observer.record( + crate::telemetry::Failure::new( + crate::telemetry::Stage::NextSlice, + error, + ) + .request(scope), + ); + })? + .ok_or(Error::BadGateway)?, + }; + let slice = match &item { + DirectItem::Plaintext(reader) => reader.slice(), + DirectItem::Sealed(_, slice) => *slice, + }; + if range.slice_at(slice.page)? != Some(slice) + || slice.length == 0 + || u64::from(slice.length) > expected - sent + { + return Err(Error::BadGateway); + } + let offset = slice + .page + .0 + .checked_mul(PAGE_BYTES) + .and_then(|n| n.checked_add(u64::from(slice.offset))) + .ok_or(Error::BadGateway)?; + // The parser and validate bound offset + range length. + let target = destination + .offset + .checked_add(offset - range.start()) + .ok_or(Error::BadGateway)?; + // A plaintext lease must outlive the write that borrows it. + let reader; + { + let (page, report) = match item { + DirectItem::Plaintext(lease) => { + reader = Some(lease); + (DirectPage::Plaintext(reader.as_ref().unwrap()), None) + } + DirectItem::Sealed(sealed, _) => { + reader = None; + let report = sealed.report(); + let start = u64::from(slice.offset); + ( + DirectPage::Sealed { + page: sealed, + range: start..start + u64::from(slice.length), + }, + Some(report), + ) + } + }; + let mut write = sink.deliver(page, target, &progress_scope); + let result = std::future::poll_fn(|cx| { + if let Poll::Ready(result) = write.as_mut().poll(cx) { + return Poll::Ready(result); + } + // Keep the fetch window moving while the write runs. + stream.poll_prefetch(cx); + Poll::Pending + }) + .await; + drop(write); + if let Some(report) = report { + // The owner drops a copy the GPU rejected, as it would + // after a failed CPU decrypt, and may persist one it + // accepted. + match result { + Ok(()) => stream.report_sealed(report, true), + Err(Error::CorruptRecord) => stream.report_sealed(report, false), + Err(_) => {} + } + } + result?; + } + drop(reader); + // The page is written: no client credit gates its release. + stream.release_page(slice.page, slice.length)?; + connection = send_frame( + &self.io, + connection, + frame(1, slice.page.0, offset, slice.length), + &progress_scope, + ) + .await?; + sent += u64::from(slice.length); + pages += 1; + } + } + let mut final_scope = scope.clone(); + final_scope.deadline.0 = uring_runtime::environment::now() + timeout; + sink.finish(&final_scope).await?; + connection = + send_frame(&self.io, connection, frame(2, pages, sent, 0), &final_scope).await?; + observation.success(); + Ok(connection) + } + /// Attach delivery resources with a no-op observer until configured. pub fn new(io: Rc, delivery: Rc) -> Self { Self { io, delivery, observer: Observer::default(), + gpu: None, } } + /// Serve `gpu-read` with this backend. Without one, `gpu-read` is 501. + pub fn with_gpu(mut self, gpu: Rc) -> Self { + self.gpu = Some(gpu); + self + } + + /// Check the descriptors that came with a request and open its sink. + pub(crate) fn open_sink( + &self, + kind: &ReadKind, + received: http1::connection::ReceivedDescriptors, + ) -> Result>> { + direct::open_sink(kind, received, &self.delivery, self.gpu.as_ref()) + } + /// Route detailed delivery failures to the worker's observer. pub(crate) fn with_observer(mut self, observer: Observer) -> Self { self.observer = observer; @@ -686,6 +922,28 @@ impl Responses { subscription_head(&response.metadata, response.payload.range())?; return Ok(()); } + ReadKind::HostRead { + range, destination, .. + } + | ReadKind::GpuRead { + range, destination, .. + } => { + let expected = expected_subscription_range(*range, response.metadata.length)?; + if response.payload.range() != expected { + return Err(Error::BadGateway); + } + // Reject before the 200 head: the window must hold the range. + if expected.map_or(0, |range| range.len()) > destination.length { + return Err(Error::InvalidRequest); + } + let delivery = kind.direct().ok_or(Error::Internal)?; + racer_object_wire::client::direct_head( + &response.metadata, + response.payload.range(), + delivery, + )?; + return Ok(()); + } ReadKind::Head | ReadKind::HeadPinned { .. } => { if !matches!(&response.payload, ReadPayload::Empty) { return Err(Error::BadGateway); @@ -769,6 +1027,7 @@ fn header(name: &str, value: impl AsRef<[u8]>) -> Header { fn error_head(error: Error) -> Result { let status = match error { Error::InvalidRequest | Error::InvalidRange => 400, + Error::NotImplemented => 501, Error::MethodNotAllowed => 405, Error::HeaderTooLarge => 431, Error::Unauthorized | Error::OriginRejected => 401, diff --git a/cmd/racer-dataplane/src/client/direct.rs b/cmd/racer-dataplane/src/client/direct.rs new file mode 100644 index 0000000000..61c648804d --- /dev/null +++ b/cmd/racer-dataplane/src/client/direct.rs @@ -0,0 +1,236 @@ +//! Direct reads write object plaintext into a client descriptor, not the socket. +//! +//! `host-read` writes into a regular file (normally a memfd) with positioned +//! io_uring writes. `gpu-read` hands each page to a [`GpuService`] backend. Both +//! share one request path through the [`DirectSink`] seam: the client code +//! resolves the range, gets each verified page from the Get read path, and asks +//! the sink to deliver it at a destination offset. The sink resolves the future +//! only when the bytes are visible to the client, so the next page frame can +//! go out. +//! +//! The client sends exactly one descriptor with the request head. Any other +//! count, a descriptor on another request, or truncated ancillary data is a 400. + +use crate::client::ReadKind; +use crate::error::Error; +use crate::error::Operation; +use crate::error::Result; +use crate::http::Delivery; +use crate::http::ReaderLease; +use crate::read::sealed::SealedPage; +use crate::read::sealed::SealedTarget; +use crate::runtime::RequestScope; +use http1::connection::ReceivedDescriptors; +pub use racer_object_wire::client::Destination; +pub use racer_object_wire::client::DirectDelivery; +use std::os::fd::AsRawFd; +use std::os::fd::OwnedFd; +use std::rc::Rc; +use uring_runtime::reactor::descriptor::Descriptor; + +/// The fewest pages the dataplane fetches ahead for one direct read. Clients +/// send no credits, so this window, or `range_window_pages` when larger, alone +/// bounds the pages a direct read pins. +pub const DIRECT_WINDOW_PAGES: usize = 4; + +/// One page handed to a [`DirectSink`]. +/// +/// Pages arrive as verified plaintext unless the sink asked for sealed pages +/// with [`DirectSink::sealed_target`]. +#[non_exhaustive] +pub enum DirectPage<'a> { + /// Verified plaintext already in memory. [`ReaderLease::slice_bytes`] holds + /// exactly the bytes to write; [`ReaderLease::page`] can be cloned to keep + /// the page alive past this call. + Plaintext(&'a ReaderLease), + + /// A whole sealed page, for the sink to verify and decrypt. Only + /// `range`, a byte range of the page's plaintext, is written. A sink + /// must not write anything if verification fails, and must then return + /// [`Error::CorruptRecord`]. + Sealed { + page: Box, + range: std::ops::Range, + }, +} + +impl DirectPage<'_> { + /// Number of bytes this page writes. + pub fn len(&self) -> u64 { + match self { + Self::Plaintext(reader) => u64::from(reader.slice().length), + Self::Sealed { range, .. } => range.end.saturating_sub(range.start), + } + } + + /// Whether this page writes nothing. Pages are never empty in practice. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// Where a direct read writes. One sink serves one request and owns the +/// client's descriptor; dropping the sink must release it. +/// +/// The caller calls [`Self::deliver`] once per page, in page order, and waits +/// for each call before the next. It calls [`Self::finish`] once after the last +/// page, and on a best-effort basis after a failure, before the kind 2 frame. +/// A sink may be dropped without `finish`, for example when the client +/// disconnects; its `Drop` must then keep any resource that a device may still +/// write rather than free it. +pub trait DirectSink { + /// Finish any setup that can still fail the request with an error status, + /// such as importing a GPU allocation. Awaited once before the read starts + /// and before the response head is sent. + fn ready<'a>(&'a mut self, _scope: &'a RequestScope) -> Operation<'a, ()> { + Box::pin(async { Ok(()) }) + } + + /// Write `page` at `destination_offset` in the client's destination. The + /// caller guarantees the bytes fall inside the request's destination + /// window. Resolve only when the bytes are visible to the client. + fn deliver<'a>( + &'a mut self, + page: DirectPage<'a>, + destination_offset: u64, + scope: &'a RequestScope, + ) -> Operation<'a, ()>; + + /// Wait for all work that touches the destination, then release it. + fn finish<'a>(&'a mut self, scope: &'a RequestScope) -> Operation<'a, ()>; + + /// Ask for sealed pages ([`DirectPage::Sealed`]) where the read path has + /// no plaintext in memory. Called after [`Self::ready`]. The default, + /// None, gets verified plaintext only. + fn sealed_target(&self) -> Option { + None + } +} + +/// A GPU backend for `gpu-read`. Without one, `gpu-read` answers 501. +pub trait GpuService { + /// Import the client's exported allocation and return a sink for one + /// request. Called after the request is validated and before any read. + /// Return [`Error::NotImplemented`] when the allocation is on a GPU the + /// dataplane cannot use; [`Error::InvalidRequest`] for a bad handle. + fn open( + &self, + descriptor: OwnedFd, + destination: Destination, + allocation_size: u64, + ) -> Result>; +} + +/// Writes pages into a client file with positioned io_uring writes. +pub(crate) struct HostSink { + file: Rc, + + delivery: Rc, + + window: Destination, +} + +impl HostSink { + /// Accept only a regular file opened for writing. Never map it, so a + /// client that truncates the file cannot fault the dataplane. + pub(crate) fn open( + descriptor: OwnedFd, + window: Destination, + delivery: Rc, + ) -> Result { + // SAFETY: fstat writes only into the zeroed stat buffer. + let mut stat: libc::stat = unsafe { std::mem::zeroed() }; + if unsafe { libc::fstat(descriptor.as_raw_fd(), &mut stat) } != 0 + || stat.st_mode & libc::S_IFMT != libc::S_IFREG + { + return Err(Error::InvalidRequest); + } + // SAFETY: F_GETFL only reads descriptor flags. + let flags = unsafe { libc::fcntl(descriptor.as_raw_fd(), libc::F_GETFL) }; + if flags < 0 || flags & libc::O_ACCMODE == libc::O_RDONLY || flags & libc::O_APPEND != 0 { + return Err(Error::InvalidRequest); + } + Ok(Self { + file: Rc::new(Descriptor::from(descriptor)), + delivery, + window, + }) + } +} + +impl DirectSink for HostSink { + fn deliver<'a>( + &'a mut self, + page: DirectPage<'a>, + destination_offset: u64, + scope: &'a RequestScope, + ) -> Operation<'a, ()> { + Box::pin(async move { + let end = destination_offset + .checked_add(page.len()) + .ok_or(Error::InvalidRange)?; + if destination_offset < self.window.offset || end > self.window.end() { + return Err(Error::InvalidRange); + } + match page { + DirectPage::Plaintext(reader) => { + self.delivery + .write_slice_at(reader, self.file.clone(), destination_offset, scope) + .await + } + // Never requested: host sinks have no sealed target. + DirectPage::Sealed { .. } => Err(Error::Internal), + } + }) + } + + fn finish<'a>(&'a mut self, _scope: &'a RequestScope) -> Operation<'a, ()> { + // Completed io_uring writes are already visible through the page cache. + Box::pin(async { Ok(()) }) + } +} + +/// Pair the descriptors received with a request head with its parsed kind, +/// and open the sink for a direct read. Unclaimed descriptors close on return. +pub(crate) fn open_sink( + kind: &ReadKind, + received: ReceivedDescriptors, + delivery: &Rc, + gpu: Option<&Rc>, +) -> Result>> { + let ReceivedDescriptors { + mut descriptors, + truncated, + } = received; + if truncated { + return Err(Error::InvalidRequest); + } + let Some(mode) = kind.direct() else { + return if descriptors.is_empty() { + Ok(None) + } else { + Err(Error::InvalidRequest) + }; + }; + let (Some(descriptor), true) = (descriptors.pop(), descriptors.is_empty()) else { + return Err(Error::InvalidRequest); + }; + let destination = kind.destination().ok_or(Error::Internal)?; + match (mode, kind) { + (DirectDelivery::Host, _) => Ok(Some(Box::new(HostSink::open( + descriptor, + destination, + delivery.clone(), + )?))), + ( + DirectDelivery::Gpu, + ReadKind::GpuRead { + allocation_size, .. + }, + ) => gpu + .ok_or(Error::NotImplemented)? + .open(descriptor, destination, *allocation_size) + .map(Some), + _ => Err(Error::Internal), + } +} diff --git a/cmd/racer-dataplane/src/client/listener.rs b/cmd/racer-dataplane/src/client/listener.rs index 8aaaf077f8..24ebef69ad 100644 --- a/cmd/racer-dataplane/src/client/listener.rs +++ b/cmd/racer-dataplane/src/client/listener.rs @@ -948,8 +948,9 @@ async fn serve_connection( )?; let idle_scope = new_scope(timeout, cancellation.clone())?; deadline.set(idle_scope.deadline.0); + // Read with recvmsg so a direct read's descriptor arrives with its head. let received = io - .receive_request_head_limited(connection, &idle_scope, parser.header_limit()) + .receive_request_head_with_descriptors(connection, &idle_scope, parser.header_limit()) .await?; connection = received.connection; if !retirement_request && retired.load(std::sync::atomic::Ordering::Acquire) { @@ -962,8 +963,21 @@ async fn serve_connection( let scope = new_scope(timeout, cancellation.clone())?; deadline.set(scope.deadline.0); let scope = &scope; - let request = match received.value.and_then(|head| parser.parse(cache, head)) { - Ok(request) => request, + // Every descriptor must be claimed by a direct read; the rest close here. + let parsed = received.value.and_then(|received| { + let request = parser.parse(cache, received.head)?; + let sink = responses.open_sink(&request.kind, received.descriptors)?; + Ok((request, sink)) + }); + let parsed = match parsed { + Ok((request, Some(mut sink))) => match sink.ready(scope).await { + Ok(()) => Ok((request, Some(sink))), + Err(error) => Err(error), + }, + other => other, + }; + let (request, sink) = match parsed { + Ok(parsed) => parsed, Err(error) => { observation.fail(error); responses.send_error(connection, error, scope).await?; @@ -1015,6 +1029,7 @@ async fn serve_connection( scope, &mut observation, timeout, + sink, ) .await? { diff --git a/cmd/racer-dataplane/src/client/tests.rs b/cmd/racer-dataplane/src/client/tests.rs index c9d00c71aa..7736b48f17 100644 --- a/cmd/racer-dataplane/src/client/tests.rs +++ b/cmd/racer-dataplane/src/client/tests.rs @@ -1095,6 +1095,7 @@ impl Fixture { &scope, &mut observation, Duration::from_secs(5), + None, ) .await?; Ok(()) @@ -1853,8 +1854,15 @@ fn client_disconnect_cancels_pending_metadata_before_acquisition_deadline() { assert_eq!(scope.check(), Ok(())); fixture.wait_for_origin(RequestKind::Head, 1); drop(socket); - for _ in 0..64 { + // The hangup completion can arrive late on a loaded host: pump until it + // is observed and the connection is gone, well inside the acquisition + // deadline. + let wait = Instant::now() + Duration::from_secs(1); + while (!scope.cancellation.is_cancelled() || fixture.listeners.active_connections() != 0) + && Instant::now() < wait + { fixture.pump(16); + std::thread::yield_now(); } assert!(scope.cancellation.is_cancelled()); assert!(Instant::now() < scope.deadline.0); @@ -4073,3 +4081,4 @@ fn handoff_acceptor_drains_backlog_and_wakes_only_while_one_remains() { } assert_eq!(worker.listeners.active_connections_for(&definition().id), 3); } +mod direct; diff --git a/cmd/racer-dataplane/src/client/tests/direct.rs b/cmd/racer-dataplane/src/client/tests/direct.rs new file mode 100644 index 0000000000..8d9c0480ae --- /dev/null +++ b/cmd/racer-dataplane/src/client/tests/direct.rs @@ -0,0 +1,544 @@ +//! `host-read` and `gpu-read` over the real client socket with a fake origin. +use super::*; +use crate::client::direct::Destination; +use crate::client::direct::DirectPage; +use crate::client::direct::DirectSink; +use crate::client::direct::GpuService; +use crate::model::CacheKey; +use crate::model::ObjectId; +use crate::model::PAGE_BYTES; +use std::io::Seek; +use std::os::fd::FromRawFd; +use std::os::fd::OwnedFd; +use std::os::fd::RawFd; + +static NAMES: AtomicUsize = AtomicUsize::new(0); + +/// A memfd with a process-unique name, so open copies can be counted. +struct Memfd { + file: File, + name: String, +} + +impl Memfd { + fn new() -> Self { + let name = format!( + "racer-direct-{}-{}", + std::process::id(), + NAMES.fetch_add(1, Ordering::Relaxed) + ); + let c_name = CString::new(name.clone()).unwrap(); + let fd = unsafe { libc::memfd_create(c_name.as_ptr(), libc::MFD_CLOEXEC) }; + assert!(fd >= 0); + Self { + file: unsafe { File::from_raw_fd(fd) }, + name, + } + } + + /// Descriptors in this process that refer to this memfd, including ours. + fn open_copies(&self) -> usize { + let exact = format!("/memfd:{} (deleted)", self.name); + fs::read_dir("/proc/self/fd") + .unwrap() + .filter_map(|entry| fs::read_link(entry.ok()?.path()).ok()) + .filter(|target| target.to_string_lossy() == exact) + .count() + } + + fn contents(&mut self) -> Vec { + let mut output = Vec::new(); + self.file.seek(std::io::SeekFrom::Start(0)).unwrap(); + self.file.read_to_end(&mut output).unwrap(); + output + } +} + +/// Send `bytes` with `fds` attached as one `SCM_RIGHTS` message. +fn send_with_fds(socket: &UnixStream, bytes: &[u8], fds: &[RawFd]) { + let payload = std::mem::size_of_val(fds); + let space = unsafe { libc::CMSG_SPACE(payload as u32) } as usize; + let mut control = vec![0u64; space.div_ceil(8)]; + let mut iov = libc::iovec { + iov_base: bytes.as_ptr().cast_mut().cast(), + iov_len: bytes.len(), + }; + let mut header: libc::msghdr = unsafe { std::mem::zeroed() }; + header.msg_iov = &mut iov; + header.msg_iovlen = 1; + if !fds.is_empty() { + header.msg_control = control.as_mut_ptr().cast(); + header.msg_controllen = space as _; + unsafe { + let cmsg = libc::CMSG_FIRSTHDR(&header); + (*cmsg).cmsg_level = libc::SOL_SOCKET; + (*cmsg).cmsg_type = libc::SCM_RIGHTS; + (*cmsg).cmsg_len = libc::CMSG_LEN(payload as u32) as _; + std::ptr::copy_nonoverlapping( + fds.as_ptr().cast::(), + libc::CMSG_DATA(cmsg), + payload, + ); + } + } + let sent = unsafe { libc::sendmsg(socket.as_raw_fd(), &header, 0) }; + assert_eq!(sent, bytes.len() as isize); +} + +fn direct_request(suffix: &str, fields: &str) -> Vec { + format!( + "POST /v1/objects/{}/{suffix} HTTP/1.1\r\nHost: racer\r\nContent-Length: 0\r\n{fields}\r\n", + "0".repeat(64) + ) + .into_bytes() +} + +/// A fixture whose fake origin serves `body` as version "v1". +fn direct_fixture(body: Vec, gpu: Option>) -> Fixture { + let mut fixture = Fixture::new(); + let delivery = Rc::new(Delivery::new( + fixture.reactor.clone(), + Duration::from_secs(2), + )); + let mut responses = Responses::new(fixture.listeners.io.clone(), delivery.clone()); + if let Some(gpu) = gpu { + responses = responses.with_gpu(gpu); + } + fixture.listeners.responses = Rc::new(responses); + let worker = ReadWorker::new( + definition(), + ObjectMetadata { + content_type: None, + version: ObjectVersion { + object: ObjectId { + cache: definition().id, + key: CacheKey([0; 32]), + }, + etag: StrongEtag::parse(b"\"v1\"").unwrap(), + }, + length: body.len() as u64, + expires_at: ExpiresAt::from_system_time(UNIX_EPOCH + Duration::from_millis(1234)) + .unwrap(), + }, + fixture.listeners.admission.clone(), + fixture.reactor.clone(), + delivery, + 1, + ); + worker.origin.set_body(body); + fixture.listeners.reads = worker.coordinator.clone(); + fixture.worker = Some(worker); + fixture.reconcile(&[definition()]).unwrap(); + fixture +} + +/// Split a complete response into its lowercased head and body bytes. +fn split(output: &[u8]) -> (String, &[u8]) { + let end = output + .windows(4) + .position(|part| part == b"\r\n\r\n") + .expect("response head") + + 4; + ( + std::str::from_utf8(&output[..end]) + .unwrap() + .to_ascii_lowercase(), + &output[end..], + ) +} + +/// Decode 21-byte frames: (kind, number, offset, length). +fn frames(body: &[u8]) -> Vec<(u8, u64, u64, u32)> { + assert_eq!(body.len() % 21, 0, "body is whole frames"); + body.chunks(21) + .map(|frame| { + ( + frame[0], + u64::from_be_bytes(frame[1..9].try_into().unwrap()), + u64::from_be_bytes(frame[9..17].try_into().unwrap()), + u32::from_be_bytes(frame[17..].try_into().unwrap()), + ) + }) + .collect() +} + +/// Send one request with `fds` and read the response to EOF. +fn exchange(fixture: &Fixture, request: &[u8], fds: &[RawFd]) -> Vec { + let mut socket = fixture.connect(); + send_with_fds(&socket, request, fds); + fixture.receive(&mut socket, true) +} + +/// Pump until the connection and its descriptors are gone. Completions can +/// arrive late on a loaded host, so wait on a deadline, not a pump count. +fn settle(fixture: &Fixture) { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + fixture.pump(16); + if fixture.listeners.active_connections() == 0 { + break; + } + assert!(Instant::now() < deadline, "connection did not close"); + std::thread::yield_now(); + } + for _ in 0..8 { + fixture.pump(16); + } +} + +#[test] +fn host_read_writes_the_range_at_the_destination_offset() { + let fixture = direct_fixture(b"hello".to_vec(), None); + let mut memfd = Memfd::new(); + memfd.file.write_all(&[b'.'; 16]).unwrap(); + let output = exchange( + &fixture, + &direct_request( + "host-read", + "Range: bytes=1-3\r\nIf-Match: \"v1\"\r\nRacer-Destination-Offset: 10\r\n\ + Racer-Destination-Length: 3\r\nRacer-Page-Credits: 1\r\n", + ), + &[memfd.file.as_raw_fd()], + ); + let (head, body) = split(&output); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert!(head.contains("racer-delivery: host\r\n"), "{head}"); + assert!(head.contains("content-length: 42\r\n"), "{head}"); + assert!(head.contains("racer-range-start: 1\r\n"), "{head}"); + assert!(head.contains("racer-range-end: 4\r\n"), "{head}"); + assert!(head.contains("connection: close\r\n"), "{head}"); + assert_eq!(frames(body), [(1, 0, 1, 3), (2, 1, 3, 0)]); + assert_eq!(memfd.contents(), b"..........ell..."); + settle(&fixture); + assert_eq!(memfd.open_copies(), 1, "dataplane closed its copy"); + assert_no_body_leases(&fixture); +} + +#[test] +fn host_read_spans_pages_and_whole_objects() { + let length = PAGE_BYTES as usize + 3; + let body: Vec = (0..length).map(|i| (i % 251) as u8).collect(); + let fixture = direct_fixture(body.clone(), None); + let mut memfd = Memfd::new(); + let output = exchange( + &fixture, + &direct_request( + "host-read", + &format!("Racer-Destination-Offset: 0\r\nRacer-Destination-Length: {length}\r\n"), + ), + &[memfd.file.as_raw_fd()], + ); + let (head, body_frames) = split(&output); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert!(head.contains("content-length: 63\r\n"), "{head}"); + assert_eq!( + frames(body_frames), + [ + (1, 0, 0, PAGE_BYTES as u32), + (1, 1, PAGE_BYTES, 3), + (2, 2, length as u64, 0) + ] + ); + assert!(memfd.contents() == body, "destination matches the object"); + settle(&fixture); + assert_eq!(memfd.open_copies(), 1); +} + +#[test] +fn host_read_descriptor_errors_are_400_and_close_descriptors() { + let fields = "Racer-Destination-Offset: 0\r\nRacer-Destination-Length: 5\r\n"; + let fixture = direct_fixture(b"hello".to_vec(), None); + let memfd = Memfd::new(); + let fd = memfd.file.as_raw_fd(); + let plain = request("POST", ""); + for (request, fds) in [ + // Missing descriptor. + (direct_request("host-read", fields), vec![]), + // Two descriptors. + (direct_request("host-read", fields), vec![fd, fd]), + // A descriptor on a plain Get. + (plain.clone(), vec![fd]), + // A descriptor on a HEAD. + (request("HEAD", ""), vec![fd]), + // Missing destination field. + ( + direct_request("host-read", "Racer-Destination-Offset: 0\r\n"), + vec![fd], + ), + ] { + let output = exchange(&fixture, &request, &fds); + let (head, body) = split(&output); + assert!( + head.starts_with("http/1.1 400"), + "{head} {:?}", + String::from_utf8_lossy(&request) + ); + assert!(body.is_empty()); + settle(&fixture); + assert_eq!( + memfd.open_copies(), + 1, + "{}", + String::from_utf8_lossy(&request) + ); + } + // Plain requests without descriptors still work. + let output = exchange(&fixture, &plain, &[]); + assert!(split(&output).0.starts_with("http/1.1 200")); +} + +#[test] +fn host_read_rejects_non_regular_and_read_only_descriptors() { + let fields = "Racer-Destination-Offset: 0\r\nRacer-Destination-Length: 5\r\n"; + let fixture = direct_fixture(b"hello".to_vec(), None); + let mut fds = [0; 2]; + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + let mut read = unsafe { File::from_raw_fd(fds[0]) }; + let write = unsafe { OwnedFd::from_raw_fd(fds[1]) }; + let output = exchange( + &fixture, + &direct_request("host-read", fields), + &[write.as_raw_fd()], + ); + assert!(split(&output).0.starts_with("http/1.1 400")); + settle(&fixture); + drop(write); + let mut byte = [0]; + assert_eq!(read.read(&mut byte).unwrap(), 0, "pipe write end leaked"); + + let memfd = Memfd::new(); + let path = format!("/proc/self/fd/{}", memfd.file.as_raw_fd()); + let read_only = File::open(path).unwrap(); + let output = exchange( + &fixture, + &direct_request("host-read", fields), + &[read_only.as_raw_fd()], + ); + assert!(split(&output).0.starts_with("http/1.1 400")); + settle(&fixture); + drop(read_only); + assert_eq!(memfd.open_copies(), 1); +} + +#[test] +fn host_read_range_and_version_errors_keep_get_statuses() { + let fixture = direct_fixture(b"hello".to_vec(), None); + let mut memfd = Memfd::new(); + for (fields, status) in [ + // Destination smaller than the range. + ( + "Range: bytes=0-4\r\nRacer-Destination-Offset: 0\r\nRacer-Destination-Length: 4\r\n", + "400", + ), + ( + "If-Match: \"v2\"\r\nRacer-Destination-Offset: 0\r\nRacer-Destination-Length: 5\r\n", + "412", + ), + ( + "Range: bytes=9-\r\nRacer-Destination-Offset: 0\r\nRacer-Destination-Length: 5\r\n", + "416", + ), + ] { + let output = exchange( + &fixture, + &direct_request("host-read", fields), + &[memfd.file.as_raw_fd()], + ); + let (head, _) = split(&output); + assert!(head.starts_with(&format!("http/1.1 {status}")), "{head}"); + settle(&fixture); + assert_eq!(memfd.open_copies(), 1, "{fields}"); + } + assert!(memfd.contents().is_empty(), "nothing was written"); + assert_no_body_leases(&fixture); +} + +#[test] +fn host_read_client_disconnect_closes_descriptor_and_releases_pages() { + let length = 2 * PAGE_BYTES as usize + 1; + let fixture = direct_fixture(vec![b'x'; length], None); + let memfd = Memfd::new(); + let mut socket = fixture.connect(); + send_with_fds( + &socket, + &direct_request( + "host-read", + &format!("Racer-Destination-Offset: 0\r\nRacer-Destination-Length: {length}\r\n"), + ), + &[memfd.file.as_raw_fd()], + ); + let output = fixture.receive(&mut socket, false); + assert!(split(&output).0.starts_with("http/1.1 200")); + drop(socket); + settle(&fixture); + assert_eq!(memfd.open_copies(), 1); + fixture.listeners.admission.reclaim_buffers(); + assert_no_body_leases(&fixture); +} + +#[test] +fn gpu_read_is_validated_then_501_without_a_backend() { + let fixture = direct_fixture(b"hello".to_vec(), None); + let memfd = Memfd::new(); + let fd = memfd.file.as_raw_fd(); + for (fields, fds, status) in [ + ( + "Racer-Destination-Offset: 0\r\nRacer-Destination-Length: 5\r\nRacer-Allocation-Size: 5\r\n", + vec![fd], + "501", + ), + // Validation precedes the 501. + ( + "Racer-Destination-Offset: 1\r\nRacer-Destination-Length: 5\r\nRacer-Allocation-Size: 5\r\n", + vec![fd], + "400", + ), + ( + "Racer-Destination-Offset: 0\r\nRacer-Destination-Length: 5\r\nRacer-Allocation-Size: 5\r\n", + vec![], + "400", + ), + ] { + let output = exchange(&fixture, &direct_request("gpu-read", fields), &fds); + let (head, body) = split(&output); + assert!(head.starts_with(&format!("http/1.1 {status}")), "{head}"); + assert!(body.is_empty()); + settle(&fixture); + assert_eq!(memfd.open_copies(), 1); + } +} + +/// Records what a GPU backend would be asked to do. +#[derive(Default)] +struct Recorded { + opened: Vec<(Destination, u64)>, + writes: Vec<(u64, Vec)>, + finished: usize, +} + +struct FakeGpu(Rc>); + +struct FakeSink(Rc>, #[allow(dead_code)] OwnedFd); + +impl GpuService for FakeGpu { + fn open( + &self, + descriptor: OwnedFd, + destination: Destination, + allocation_size: u64, + ) -> Result> { + self.0 + .borrow_mut() + .opened + .push((destination, allocation_size)); + Ok(Box::new(FakeSink(self.0.clone(), descriptor))) + } +} + +impl DirectSink for FakeSink { + fn deliver<'a>( + &'a mut self, + page: DirectPage<'a>, + destination_offset: u64, + _scope: &'a RequestScope, + ) -> crate::error::Operation<'a, ()> { + let DirectPage::Plaintext(reader) = page else { + panic!("fake sink asks for plaintext only"); + }; + let bytes = reader.slice_bytes().to_vec(); + let recorded = self.0.clone(); + Box::pin(async move { + // Complete on a later poll, as a helper thread would. + let mut ready = false; + std::future::poll_fn(|cx| { + if std::mem::replace(&mut ready, true) { + Poll::Ready(()) + } else { + cx.waker().wake_by_ref(); + Poll::Pending + } + }) + .await; + recorded + .borrow_mut() + .writes + .push((destination_offset, bytes)); + Ok(()) + }) + } + + fn finish<'a>(&'a mut self, _scope: &'a RequestScope) -> crate::error::Operation<'a, ()> { + self.0.borrow_mut().finished += 1; + Box::pin(async { Ok(()) }) + } +} + +#[test] +fn gpu_read_delivers_pages_through_the_backend_seam() { + let recorded = Rc::new(RefCell::new(Recorded::default())); + let fixture = direct_fixture(b"hello".to_vec(), Some(Rc::new(FakeGpu(recorded.clone())))); + let memfd = Memfd::new(); + let output = exchange( + &fixture, + &direct_request( + "gpu-read", + "Range: bytes=2-\r\nRacer-Destination-Offset: 100\r\n\ + Racer-Destination-Length: 3\r\nRacer-Allocation-Size: 4096\r\n", + ), + &[memfd.file.as_raw_fd()], + ); + let (head, body) = split(&output); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert!(head.contains("racer-delivery: gpu\r\n"), "{head}"); + assert_eq!(frames(body), [(1, 0, 2, 3), (2, 1, 3, 0)]); + { + let recorded = recorded.borrow(); + assert_eq!( + recorded.opened, + [( + Destination { + offset: 100, + length: 3 + }, + 4096 + )] + ); + assert_eq!(recorded.writes, [(100, b"llo".to_vec())]); + assert_eq!(recorded.finished, 1); + } + settle(&fixture); + assert_eq!(memfd.open_copies(), 1); +} + +mod gpu; + +/// A plain HEAD pipelined ahead of a host-read on one connection: one receive +/// can return both heads with the host-read's descriptor. The descriptor must +/// not be attached to the HEAD, so the HEAD is served. The http layer rejects +/// pipelining at exchange completion, so the connection then closes without +/// serving the host-read and closes its carried descriptor. +#[test] +fn pipelined_head_then_host_read_serves_the_head_and_closes_the_descriptor() { + let fixture = direct_fixture(b"hello".to_vec(), None); + let mut memfd = Memfd::new(); + memfd.file.write_all(&[b'.'; 8]).unwrap(); + let mut socket = fixture.connect(); + socket.write_all(&request("HEAD", "")).unwrap(); + send_with_fds( + &socket, + &direct_request( + "host-read", + "Racer-Destination-Offset: 0\r\nRacer-Destination-Length: 5\r\n", + ), + &[memfd.file.as_raw_fd()], + ); + let output = fixture.receive(&mut socket, true); + let (head, rest) = split(&output); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert!(head.contains("content-length: 5\r\n"), "{head}"); + assert!(rest.is_empty(), "pipelined request is not served"); + assert_eq!(memfd.contents(), b"........"); + settle(&fixture); + assert_eq!(memfd.open_copies(), 1, "dataplane closed its copy"); + assert_no_body_leases(&fixture); +} diff --git a/cmd/racer-dataplane/src/client/tests/direct/gpu.rs b/cmd/racer-dataplane/src/client/tests/direct/gpu.rs new file mode 100644 index 0000000000..3fd10e79e6 --- /dev/null +++ b/cmd/racer-dataplane/src/client/tests/direct/gpu.rs @@ -0,0 +1,659 @@ +//! `gpu-read` end to end over the client socket, on simulated GPUs. +use super::*; +use crate::config::GpuMode; +use crate::gpu::GpuClient; +use crate::gpu::GpuHelper; +use crate::telemetry::Event; +use racer_gpu::simulation::Environment; +use racer_gpu::simulation::Live; +use racer_gpu::simulation::Operation; +use racer_gpu::simulation::SimDevice; +use racer_gpu::simulation::Simulation; +use racer_gpu::simulation::read_allocation; + +const GRANULE: u64 = racer_gpu::simulation::GRANULARITY; + +thread_local! { + /// The simulation, on the GPU helper thread only. + static SIM: RefCell> = const { RefCell::new(None) }; +} + +fn sim() -> Simulation { + SIM.with(|sim| sim.borrow().as_ref().unwrap().0.clone()) +} + +/// A helper backed by `count` simulated GPUs. +fn start(mode: GpuMode, count: u8) -> Option> { + let devices: Vec = (0..count).map(SimDevice::new).collect(); + GpuHelper::start_with( + mode, + Box::new(move || { + let simulation = Simulation::new(devices); + let environment = simulation.enter(); + SIM.with(|sim| *sim.borrow_mut() = Some((simulation, environment))); + racer_gpu::Gpu::load() + }), + ) + .unwrap() +} + +fn allocate(helper: &GpuHelper, ordinal: i32, size: u64) -> OwnedFd { + helper.run(move || sim().allocate(ordinal, size)) +} + +fn live(helper: &GpuHelper) -> Live { + helper.run(|| sim().live()) +} + +/// No request holds GPU resources. Contexts stay retained by the helper. +fn assert_released(helper: &GpuHelper) { + let live = live(helper); + assert_eq!((live.imports, live.streams), (0, 0), "{live:?}"); + assert_eq!(helper.open_requests(), 0); + assert_eq!(helper.leaked(), 0); +} + +fn gpu_fixture(body: Vec, helper: &Arc) -> Fixture { + direct_fixture(body, Some(Rc::new(GpuClient::new(helper.clone())))) +} + +fn fields(range: &str, offset: u64, length: u64, size: u64) -> String { + format!( + "{range}Racer-Destination-Offset: {offset}\r\nRacer-Destination-Length: {length}\r\n\ + Racer-Allocation-Size: {size}\r\n" + ) +} + +#[test] +fn gpu_read_writes_the_range_at_the_destination_offset() { + let helper = start(GpuMode::Required, 2).unwrap(); + assert_eq!(helper.devices().len(), 2); + let fixture = gpu_fixture(b"hello".to_vec(), &helper); + // The allocation is on the second GPU, so the helper must find it. + let allocation = allocate(&helper, 1, GRANULE); + let output = exchange( + &fixture, + &direct_request( + "gpu-read", + &fields("Range: bytes=1-3\r\nIf-Match: \"v1\"\r\n", 100, 3, GRANULE), + ), + &[allocation.as_raw_fd()], + ); + let (head, body) = split(&output); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert!(head.contains("racer-delivery: gpu\r\n"), "{head}"); + assert!(head.contains("content-length: 42\r\n"), "{head}"); + assert_eq!(frames(body), [(1, 0, 1, 3), (2, 1, 3, 0)]); + assert_eq!(read_allocation(&allocation, 99, 5), b"\0ell\0"); + settle(&fixture); + assert_released(&helper); + assert_no_body_leases(&fixture); +} + +#[test] +fn gpu_read_spans_pages() { + let length = 2 * PAGE_BYTES as usize + 7; + let body: Vec = (0..length).map(|i| (i % 251) as u8).collect(); + let helper = start(GpuMode::Auto, 1).unwrap(); + let fixture = gpu_fixture(body.clone(), &helper); + let size = (length as u64 + 10).next_multiple_of(GRANULE); + let allocation = allocate(&helper, 0, size); + let output = exchange( + &fixture, + &direct_request("gpu-read", &fields("", 10, length as u64, size)), + &[allocation.as_raw_fd()], + ); + let (head, body_frames) = split(&output); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert_eq!( + frames(body_frames), + [ + (1, 0, 0, PAGE_BYTES as u32), + (1, 1, PAGE_BYTES, PAGE_BYTES as u32), + (1, 2, 2 * PAGE_BYTES, 7), + (2, 3, length as u64, 0) + ] + ); + assert!(read_allocation(&allocation, 10, length) == body); + settle(&fixture); + assert_released(&helper); +} + +#[test] +fn gpu_read_errors_before_the_head_release_everything() { + let helper = start(GpuMode::Required, 1).unwrap(); + let fixture = gpu_fixture(b"hello".to_vec(), &helper); + let allocation = allocate(&helper, 0, GRANULE); + // On a GPU the dataplane did not discover. + let foreign = allocate(&helper, 3, GRANULE); + for (fields, fd, status) in [ + ( + fields("If-Match: \"v2\"\r\n", 0, 5, GRANULE), + &allocation, + "412", + ), + // Larger than the real allocation. + (fields("", 0, 5, 2 * GRANULE), &allocation, "400"), + // Not a whole number of granules. + (fields("", 0, 5, GRANULE - 1), &allocation, "400"), + (fields("", 0, 5, GRANULE), &foreign, "501"), + ] { + let output = exchange( + &fixture, + &direct_request("gpu-read", &fields), + &[fd.as_raw_fd()], + ); + let (head, body) = split(&output); + assert!( + head.starts_with(&format!("http/1.1 {status}")), + "{fields} {head}" + ); + assert!(body.is_empty()); + settle(&fixture); + assert_released(&helper); + } + assert_eq!( + read_allocation(&allocation, 0, 5), + [0; 5], + "nothing written" + ); + assert_no_body_leases(&fixture); +} + +#[test] +fn gpu_read_driver_fault_fails_the_read_and_releases_the_import() { + let length = 2 * PAGE_BYTES as usize; + let helper = start(GpuMode::Required, 1).unwrap(); + let fixture = gpu_fixture(vec![b'x'; length], &helper); + let size = (length as u64).next_multiple_of(GRANULE); + let allocation = allocate(&helper, 0, size); + helper.run(|| sim().fault(Operation::CopyH2d, -5)); + let output = exchange( + &fixture, + &direct_request("gpu-read", &fields("", 0, length as u64, size)), + &[allocation.as_raw_fd()], + ); + let (head, body) = split(&output); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert!( + frames(body).iter().all(|frame| frame.0 != 2), + "no kind 2 frame after a failure" + ); + settle(&fixture); + assert_released(&helper); + fixture.listeners.admission.reclaim_buffers(); + assert_no_body_leases(&fixture); +} + +#[test] +fn gpu_read_client_disconnect_releases_the_import() { + let length = 3 * PAGE_BYTES as usize + 1; + let helper = start(GpuMode::Required, 1).unwrap(); + let fixture = gpu_fixture(vec![b'x'; length], &helper); + let size = (length as u64).next_multiple_of(GRANULE); + let allocation = allocate(&helper, 0, size); + let mut socket = fixture.connect(); + send_with_fds( + &socket, + &direct_request("gpu-read", &fields("", 0, length as u64, size)), + &[allocation.as_raw_fd()], + ); + let output = fixture.receive(&mut socket, false); + assert!(split(&output).0.starts_with("http/1.1 200")); + drop(socket); + settle(&fixture); + assert_released(&helper); + fixture.listeners.admission.reclaim_buffers(); + assert_no_body_leases(&fixture); +} + +#[test] +fn gpu_mode_resolves_against_hardware() { + assert!(start(GpuMode::Disabled, 1).is_none()); + assert!(start(GpuMode::Auto, 0).is_none()); + assert!( + GpuHelper::start_with( + GpuMode::Required, + Box::new(|| Err(racer_gpu::Error::Unavailable)) + ) + .is_err() + ); + let unusable = SimDevice { + compute_mode: 3, + ..SimDevice::new(0) + }; + assert!( + GpuHelper::start_with( + GpuMode::Required, + Box::new(move || { + let simulation = Simulation::new(vec![unusable]); + std::mem::forget(simulation.enter()); + racer_gpu::Gpu::load() + }) + ) + .is_err() + ); + // Without the cuda feature, auto never loads a backend. + if !cfg!(feature = "cuda") { + assert!( + GpuHelper::start(GpuMode::Auto, crate::config::DEFAULT_GPU_STAGING_BYTES) + .unwrap() + .is_none() + ); + } +} + +/// A helper on one simulated GPU with a staging cap, after `setup` runs on +/// the fresh simulation. +fn start_with(staging: u64, setup: fn(&Simulation)) -> Arc { + GpuHelper::start_with_staging( + GpuMode::Required, + staging, + Box::new(move || { + let simulation = Simulation::new(vec![SimDevice::new(0)]); + let environment = simulation.enter(); + setup(&simulation); + SIM.with(|sim| *sim.borrow_mut() = Some((simulation, environment))); + racer_gpu::Gpu::load() + }), + ) + .unwrap() + .unwrap() +} + +fn page_id(number: u64) -> crate::model::PageId { + crate::model::PageId { + version: ObjectVersion { + object: ObjectId { + cache: definition().id, + key: CacheKey([0; 32]), + }, + etag: StrongEtag::parse(b"\"v1\"").unwrap(), + }, + number: crate::model::PageNumber(number), + } +} + +fn worker(fixture: &Fixture) -> &crate::test_support::ReadWorker { + fixture.worker.as_ref().unwrap() +} + +/// Calls to `operation` in the trace from `from` on. +fn calls(helper: &GpuHelper, from: usize, operation: Operation) -> usize { + helper.run(move || { + sim().trace()[from..] + .iter() + .filter(|event| event.operation == operation) + .count() + }) +} + +fn trace_len(helper: &GpuHelper) -> usize { + helper.run(|| sim().trace().len()) +} + +fn counter(value: &std::sync::atomic::AtomicU64) -> u64 { + value.load(std::sync::atomic::Ordering::Relaxed) +} + +/// A response head, its frames, the allocation and its size. +type Read = (String, Vec<(u8, u64, u64, u32)>, OwnedFd, u64); + +/// Read `range` of the object into a fresh allocation at offset 64. +fn gpu_read(fixture: &Fixture, helper: &GpuHelper, range: Option<(u64, u64)>, length: u64) -> Read { + let size = (length + 64).next_multiple_of(GRANULE); + let allocation = allocate(helper, 0, size); + let header = range.map_or(String::new(), |(start, end)| { + format!("Range: bytes={start}-{end}\r\n") + }); + let output = exchange( + fixture, + &direct_request("gpu-read", &fields(&header, 64, length, size)), + &[allocation.as_raw_fd()], + ); + let (head, body) = split(&output); + let frames = frames(body); + settle(fixture); + (head, frames, allocation, size) +} + +/// Fill the page cache with an ordinary read, then leave each page's +/// ciphertext in memory, on disk, or both, and no plaintext. +fn arrange( + fixture: &Fixture, + helper: &GpuHelper, + pages: u64, + length: u64, + memory: bool, + disk: bool, +) { + let worker = worker(fixture); + worker.retain(true); + if disk { + worker.open_disk(); + } + let (head, _, _, _) = gpu_read(fixture, helper, None, length); + assert!(head.starts_with("http/1.1 200"), "{head}"); + let mut copies = Vec::new(); + for number in 0..pages { + let copy = worker + .memory() + .ciphertext(&page_id(number)) + .unwrap() + .unwrap(); + copies.push(copy); + } + worker.memory().remove_cache(&definition().id).unwrap(); + for copy in copies { + if disk { + worker.persist(copy.clone()); + } + if memory { + worker + .memory() + .publish_ciphertext(crate::memory::UnverifiedPage { + copy, + disk_token: None, + }) + .unwrap(); + } + } +} + +/// Nothing is held: no imports, streams, leaks or staged slots. +fn assert_idle(fixture: &Fixture, helper: &GpuHelper, slots: usize) { + settle(fixture); + assert_released(helper); + assert_eq!(helper.free_slots(0), slots); +} + +fn body(length: usize) -> Vec { + (0..length).map(|i| (i * 7 % 251) as u8).collect() +} + +/// Slots left for staged ciphertext with the default staging cap. +fn default_slots() -> usize { + (crate::config::DEFAULT_GPU_STAGING_BYTES / crate::gpu::SLOT_BYTES) as usize - 2 +} + +/// The memory cache keeps only ciphertext, so a page read before is +/// decrypted again, on the GPU, from the cached ciphertext. +#[test] +fn gpu_read_memory_ciphertext_hit_decrypts_on_the_gpu() { + let helper = start(GpuMode::Required, 1).unwrap(); + let data = body(5000); + let fixture = gpu_fixture(data.clone(), &helper); + worker(&fixture).retain(true); + let _ = gpu_read(&fixture, &helper, None, 5000); + assert!( + worker(&fixture) + .memory() + .get(&page_id(0)) + .unwrap() + .is_none() + ); + assert!( + worker(&fixture) + .memory() + .ciphertext(&page_id(0)) + .unwrap() + .is_some() + ); + let mark = trace_len(&helper); + let decrypts = worker(&fixture).cpu_decrypts(); + let decrypted = counter(&helper.counters().decrypted); + let (head, frames, allocation, _) = gpu_read(&fixture, &helper, Some((10, 99)), 90); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert_eq!(frames, [(1, 0, 10, 90), (2, 1, 90, 0)]); + assert_eq!(read_allocation(&allocation, 64, 90), &data[10..100]); + assert_eq!(calls(&helper, mark, Operation::FileRead), 0); + assert_eq!(calls(&helper, mark, Operation::CopyH2d), 1); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 1); + assert_eq!(calls(&helper, mark, Operation::CopyD2d), 1); + assert_eq!(worker(&fixture).cpu_decrypts(), decrypts); + assert_eq!(counter(&helper.counters().decrypted), decrypted + 1); + assert_eq!(worker(&fixture).metric(Event::GpuDirectDiskRead), 0); + assert_idle(&fixture, &helper, default_slots()); +} + +/// A page cached in memory and on disk is staged from memory: cuFile does +/// not read the disk copy. +#[test] +fn gpu_read_prefers_memory_ciphertext_over_cufile() { + let helper = start(GpuMode::Required, 1).unwrap(); + let data = body(5000); + let fixture = gpu_fixture(data.clone(), &helper); + arrange(&fixture, &helper, 1, 5000, true, true); + assert!(worker(&fixture).disk_location(&page_id(0)).is_some()); + let mark = trace_len(&helper); + let (head, frames, allocation, _) = gpu_read(&fixture, &helper, None, 5000); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert_eq!(frames.len(), 2, "{frames:?}"); + assert_eq!(read_allocation(&allocation, 64, 5000), data); + assert_eq!(calls(&helper, mark, Operation::FileRead), 0); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 1); + assert_eq!(worker(&fixture).cpu_decrypts(), 0); + assert_eq!(worker(&fixture).metric(Event::GpuDirectDiskRead), 0); + assert_idle(&fixture, &helper, default_slots()); +} + +#[test] +fn gpu_read_retained_ciphertext_decrypts_part_of_a_page_on_the_gpu() { + let helper = start(GpuMode::Required, 1).unwrap(); + let data = body(5000); + let fixture = gpu_fixture(data.clone(), &helper); + arrange(&fixture, &helper, 1, 5000, true, false); + let mark = trace_len(&helper); + let (head, frames, allocation, size) = gpu_read(&fixture, &helper, Some((100, 199)), 100); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert_eq!(frames, [(1, 0, 100, 100), (2, 1, 100, 0)]); + // Only the window holds bytes, though the GPU decrypted the whole page. + let all = read_allocation(&allocation, 0, size as usize); + assert_eq!(&all[64..164], &data[100..200]); + assert!(all[..64].iter().chain(&all[164..]).all(|&b| b == 0)); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 1); + assert_eq!(calls(&helper, mark, Operation::CopyD2d), 1); + assert_eq!(worker(&fixture).cpu_decrypts(), 0); + assert_eq!(counter(&helper.counters().decrypted), 1); + assert_idle(&fixture, &helper, default_slots()); +} + +#[test] +fn gpu_read_disk_pages_go_through_cufile_and_gpu_decrypt() { + let length = 2 * PAGE_BYTES as usize + 4099; + let helper = start(GpuMode::Required, 1).unwrap(); + let data = body(length); + let fixture = gpu_fixture(data.clone(), &helper); + arrange(&fixture, &helper, 3, length as u64, false, true); + let mark = trace_len(&helper); + let (head, frames, allocation, _) = gpu_read(&fixture, &helper, None, length as u64); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert_eq!(frames.len(), 4, "{frames:?}"); + assert_eq!(frames[3], (2, 3, length as u64, 0)); + assert!(read_allocation(&allocation, 64, length) == data); + assert_eq!(calls(&helper, mark, Operation::FileRead), 3); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 3); + // Whole pages decrypt straight into the import. + assert_eq!(calls(&helper, mark, Operation::CopyD2d), 0); + assert_eq!(calls(&helper, mark, Operation::CopyH2d), 0); + assert_eq!(worker(&fixture).cpu_decrypts(), 0); + assert_eq!(counter(&helper.counters().staged), 3); + assert_eq!(counter(&helper.counters().decrypted), 3); + assert_eq!(worker(&fixture).metric(Event::GpuDirectDiskRead), 3); + assert!(worker(&fixture).metric(Event::GpuDirectDiskReadBytes) >= length as u64); + assert_idle(&fixture, &helper, default_slots()); +} + +#[test] +fn gpu_read_tampered_disk_ciphertext_fails_and_drops_the_record() { + use std::os::unix::fs::FileExt; + let helper = start(GpuMode::Required, 1).unwrap(); + let data = body(5000); + let fixture = gpu_fixture(data, &helper); + arrange(&fixture, &helper, 1, 5000, false, true); + let location = worker(&fixture).disk_location(&page_id(0)).unwrap(); + let slab = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(worker(&fixture).slab_path()) + .unwrap(); + let mut record = vec![0; location.extent.length()]; + slab.read_exact_at(&mut record, location.extent.offset()) + .unwrap(); + let decoded = crate::store::parse_bytes(&record, location.extent).unwrap(); + let at = location.extent.offset() + decoded.ciphertext.start as u64 + 10; + let mut byte = [0]; + slab.read_exact_at(&mut byte, at).unwrap(); + slab.write_all_at(&[byte[0] ^ 1], at).unwrap(); + slab.sync_all().unwrap(); + let (head, frames, allocation, size) = gpu_read(&fixture, &helper, None, 5000); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert!(frames.is_empty(), "{frames:?}"); + assert!( + read_allocation(&allocation, 0, size as usize) + .iter() + .all(|&b| b == 0) + ); + assert_eq!(counter(&helper.counters().rejected), 1); + // The owner dropped the record, as after a failed CPU decrypt. + assert!(worker(&fixture).disk_location(&page_id(0)).is_none()); + assert_eq!(worker(&fixture).cpu_decrypts(), 0); + assert_idle(&fixture, &helper, default_slots()); +} + +/// Replace the retained ciphertext of page 0 with tampered bytes, keeping +/// the original CRC (`keep_crc`) or the CRC of the tampered bytes. +fn tamper_retained(fixture: &Fixture, keep_crc: bool) { + use crate::memory::CiphertextBytes; + use crate::memory::CiphertextPage; + let memory = worker(fixture).memory(); + let copy = memory.unverified(&page_id(0)).unwrap().unwrap().copy; + let mut bytes = copy.ciphertext.bytes().to_vec(); + bytes[3] ^= 0x40; + let crc = if keep_crc { + copy.ciphertext.checksum() + } else { + racer_crypto::crc64(&bytes) + }; + let reservation = fixture + .listeners + .admission + .reserve( + Some(&definition().id), + crate::admission::ResourceClass::Ciphertext, + bytes.len(), + ) + .unwrap(); + let tampered = CiphertextPage { + provenance: None, + inner: Arc::new(CiphertextBytes { + checksum: std::sync::OnceLock::from(crc), + envelope: copy.ciphertext.envelope().clone(), + storage: flow_control::ChargedBytes { bytes, reservation }.into(), + }), + }; + memory.remove_cache(&definition().id).unwrap(); + memory + .publish_ciphertext(crate::memory::UnverifiedPage { + copy: crate::memory::CiphertextCopy { + metadata: copy.metadata, + ciphertext: tampered, + }, + disk_token: None, + }) + .unwrap(); +} + +#[test] +fn gpu_read_crc_and_tag_mismatches_fail_without_writing() { + for keep_crc in [true, false] { + let helper = start(GpuMode::Required, 1).unwrap(); + let fixture = gpu_fixture(body(5000), &helper); + arrange(&fixture, &helper, 1, 5000, true, false); + tamper_retained(&fixture, keep_crc); + let mark = trace_len(&helper); + let (head, frames, allocation, size) = gpu_read(&fixture, &helper, Some((0, 9)), 10); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert!(frames.is_empty(), "{frames:?}"); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 1); + // A failed verdict copies nothing out of scratch. + assert_eq!(calls(&helper, mark, Operation::CopyD2d), 0); + assert!( + read_allocation(&allocation, 0, size as usize) + .iter() + .all(|&b| b == 0) + ); + assert_eq!(counter(&helper.counters().rejected), 1); + // The rejected copy left memory. + assert!( + worker(&fixture) + .memory() + .unverified(&page_id(0)) + .unwrap() + .is_none() + ); + assert_idle(&fixture, &helper, default_slots()); + } +} + +#[test] +fn gpu_read_without_free_staging_reads_disk_through_host_memory() { + // Room for the two scratch slots only. + let staging = (2 * crate::gpu::SLOT_BYTES).next_multiple_of(crate::config::GPU_STAGING_UNIT); + let helper = start_with(staging, |_| {}); + let data = body(5000); + let fixture = gpu_fixture(data.clone(), &helper); + arrange(&fixture, &helper, 1, 5000, false, true); + let mark = trace_len(&helper); + let (head, frames, allocation, _) = gpu_read(&fixture, &helper, None, 5000); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert_eq!(frames, [(1, 0, 0, 5000), (2, 1, 5000, 0)]); + assert_eq!(read_allocation(&allocation, 64, 5000), data); + assert_eq!(calls(&helper, mark, Operation::FileRead), 0); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 1); + assert!(counter(&helper.counters().exhausted) >= 1); + assert_eq!(worker(&fixture).cpu_decrypts(), 0); + assert_idle(&fixture, &helper, 0); +} + +#[test] +fn gpu_read_falls_back_to_host_ciphertext_when_cufile_fails() { + let helper = start(GpuMode::Required, 1).unwrap(); + let data = body(5000); + let fixture = gpu_fixture(data.clone(), &helper); + arrange(&fixture, &helper, 1, 5000, false, true); + helper.run(|| sim().fault(Operation::FileRead, -5)); + let mark = trace_len(&helper); + let (head, frames, allocation, _) = gpu_read(&fixture, &helper, None, 5000); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert_eq!(frames, [(1, 0, 0, 5000), (2, 1, 5000, 0)]); + assert_eq!(read_allocation(&allocation, 64, 5000), data); + assert_eq!(calls(&helper, mark, Operation::FileRead), 1); + assert_eq!(calls(&helper, mark, Operation::CopyH2d), 1); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 1); + assert_eq!(counter(&helper.counters().file_failures), 1); + assert_eq!(worker(&fixture).metric(Event::GpuDirectDiskRead), 0); + assert_eq!(worker(&fixture).metric(Event::GpuDirectDiskReadBytes), 0); + assert_eq!(worker(&fixture).cpu_decrypts(), 0); + assert_idle(&fixture, &helper, default_slots()); +} + +#[test] +fn gpu_read_without_decrypt_kernels_decrypts_on_the_cpu() { + let helper = start_with(crate::config::DEFAULT_GPU_STAGING_BYTES, |simulation| { + simulation.disable_kernels() + }); + assert!(!helper.decrypts(0)); + let data = body(5000); + let fixture = gpu_fixture(data.clone(), &helper); + arrange(&fixture, &helper, 1, 5000, false, true); + let mark = trace_len(&helper); + let (head, frames, allocation, _) = gpu_read(&fixture, &helper, None, 5000); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert_eq!(frames, [(1, 0, 0, 5000), (2, 1, 5000, 0)]); + assert_eq!(read_allocation(&allocation, 64, 5000), data); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 0); + assert_eq!(calls(&helper, mark, Operation::FileRead), 0); + assert_eq!(worker(&fixture).cpu_decrypts(), 1); + assert_idle(&fixture, &helper, default_slots()); +} diff --git a/cmd/racer-dataplane/src/config.rs b/cmd/racer-dataplane/src/config.rs index f786f0be5e..61b594baae 100644 --- a/cmd/racer-dataplane/src/config.rs +++ b/cmd/racer-dataplane/src/config.rs @@ -121,6 +121,15 @@ pub struct Config { /// Opt into logical-CPU sizing instead of one eligible CPU per physical core. pub allow_smt: bool, pub enable_rdma: bool, + /// `gpu-read` support. Resolved against hardware at startup, on the GPU + /// helper thread, because GPU handles cannot move between threads. + pub enable_gpu: GpuMode, + /// Per-GPU cap on dataplane-owned ciphertext staging memory. Parsed and + /// validated now; plaintext `gpu-read` does not allocate staging. + pub gpu_staging_bytes: u64, + /// Per-GPU GPU memory that peers' NICs write `gpu-read` ciphertext into + /// over RDMA (GPUDirect RDMA landing). 0 turns landing off. + pub gpu_landing_bytes: u64, /// Opt into experimental opaque HTTP transit; materialized relay is the default. pub opaque_relay: bool, /// Peer-only TCP_NODELAY. Peer exchanges are strictly request/response with @@ -187,6 +196,22 @@ impl Config { "false" => false, _ => return Err(Error::InvalidConfiguration), }; + let enable_gpu = match raw.enable_gpu.as_deref().unwrap_or("auto") { + "auto" => GpuMode::Auto, + "true" => GpuMode::Required, + "false" => GpuMode::Disabled, + _ => return Err(Error::InvalidConfiguration), + }; + let gpu_staging_bytes = raw.gpu_staging_bytes.unwrap_or(DEFAULT_GPU_STAGING_BYTES); + if gpu_staging_bytes < GPU_STAGING_UNIT + || !gpu_staging_bytes.is_multiple_of(GPU_STAGING_UNIT) + { + return Err(Error::InvalidConfiguration); + } + let gpu_landing_bytes = raw.gpu_landing_bytes.unwrap_or(DEFAULT_GPU_LANDING_BYTES); + if !gpu_landing_bytes.is_multiple_of(GPU_STAGING_UNIT) { + return Err(Error::InvalidConfiguration); + } let mut config = Self { warm: crate::read::warm::Config { max_ttl: Duration::from_secs(raw.warm_max_ttl_secs.unwrap_or(3600)), @@ -239,6 +264,9 @@ impl Config { crypto_helpers: raw.crypto_helpers.map(to_usize).transpose()?, allow_smt: raw.allow_smt.unwrap_or(false), enable_rdma, + enable_gpu, + gpu_staging_bytes, + gpu_landing_bytes, opaque_relay: raw.opaque_relay.unwrap_or(false), peer_tcp_nodelay: raw.peer_tcp_nodelay.unwrap_or(true), memory_pressure: raw.memory_pressure.unwrap_or(true), @@ -580,11 +608,35 @@ impl Config { } } +/// How `RACER_ENABLE_GPU` asks for `gpu-read`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GpuMode { + /// Serve `gpu-read` when built with `cuda` and a usable GPU is found. + Auto, + /// Fail startup unless a usable GPU is found. + Required, + /// Never serve `gpu-read`; answer 501. + Disabled, +} + +/// Default `RACER_GPU_STAGING_BYTES`: 256 MiB per GPU. +pub const DEFAULT_GPU_STAGING_BYTES: u64 = 256 * 1024 * 1024; + +/// Default `RACER_GPU_LANDING_BYTES`: 256 MiB per GPU. +pub const DEFAULT_GPU_LANDING_BYTES: u64 = 256 * 1024 * 1024; + +/// GPU staging is allocated in 2 MiB device granules, so the cap must be a +/// whole number of them. +pub const GPU_STAGING_UNIT: u64 = 2 * 1024 * 1024; + /// Environment names are exact and case sensitive, even though envy folds keys. const ENV_NAMES: &[&str] = &[ "CLUSTER_ID", "CONTROL_ENDPOINT", "ENABLE_RDMA", + "ENABLE_GPU", + "GPU_STAGING_BYTES", + "GPU_LANDING_BYTES", "ALLOW_SMT", "OPAQUE_RELAY", "PEER_TCP_NODELAY", @@ -660,6 +712,11 @@ struct RawConfig { cluster_id: String, control_endpoint: String, enable_rdma: Option, + enable_gpu: Option, + #[serde(default, deserialize_with = "decimal")] + gpu_staging_bytes: Option, + #[serde(default, deserialize_with = "decimal")] + gpu_landing_bytes: Option, allow_smt: Option, opaque_relay: Option, peer_tcp_nodelay: Option, @@ -1323,6 +1380,63 @@ mod tests { assert!(parse(&[("RACER_ENABLE_RDMA", "true")]).unwrap().enable_rdma); } + #[test] + fn gpu_mode_and_staging_bytes_parse_and_validate() { + let config = parse(&[]).unwrap(); + assert_eq!(config.enable_gpu, GpuMode::Auto); + assert_eq!(config.gpu_staging_bytes, 256 << 20); + for (value, mode) in [ + ("auto", GpuMode::Auto), + ("true", GpuMode::Required), + ("false", GpuMode::Disabled), + ] { + assert_eq!( + parse(&[("RACER_ENABLE_GPU", value)]).unwrap().enable_gpu, + mode + ); + } + for value in ["TRUE", "1", "yes", "", " auto"] { + assert!(parse(&[("RACER_ENABLE_GPU", value)]).is_err(), "{value:?}"); + } + assert_eq!( + parse(&[("RACER_GPU_STAGING_BYTES", "4194304")]) + .unwrap() + .gpu_staging_bytes, + 4 << 20 + ); + for value in [ + "0", + "1048576", + "3145728", + "-2097152", + "2MiB", + "18446744073709551616", + ] { + assert!( + parse(&[("RACER_GPU_STAGING_BYTES", value)]).is_err(), + "{value:?}" + ); + } + assert_eq!( + parse(&[]).unwrap().gpu_landing_bytes, + DEFAULT_GPU_LANDING_BYTES + ); + for (value, bytes) in [("0", 0), ("4194304", 4 << 20)] { + assert_eq!( + parse(&[("RACER_GPU_LANDING_BYTES", value)]) + .unwrap() + .gpu_landing_bytes, + bytes + ); + } + for value in ["1048576", "3145728", "-2097152", "2MiB"] { + assert!( + parse(&[("RACER_GPU_LANDING_BYTES", value)]).is_err(), + "{value:?}" + ); + } + } + #[test] fn native_auto_requires_build_and_available_library() { for requested in [None, Some("auto"), Some("false")] { diff --git a/cmd/racer-dataplane/src/error.rs b/cmd/racer-dataplane/src/error.rs index c16901a5a6..05be6fc6cf 100644 --- a/cmd/racer-dataplane/src/error.rs +++ b/cmd/racer-dataplane/src/error.rs @@ -67,6 +67,8 @@ boundary_errors! { CorruptRecord, MissingKey, DirectIoUnsupported, + /// The request is valid, but this dataplane cannot serve it (HTTP 501). + NotImplemented, Io, Os(errno: i32), } diff --git a/cmd/racer-dataplane/src/gpu.rs b/cmd/racer-dataplane/src/gpu.rs new file mode 100644 index 0000000000..39570e52a3 --- /dev/null +++ b/cmd/racer-dataplane/src/gpu.rs @@ -0,0 +1,1909 @@ +//! `gpu-read` backend: one GPU helper thread owns every CUDA object. +//! +//! `racer_gpu` types are `!Send`, so all GPU state lives on a single native +//! helper thread started once per process. I/O threads talk to it through a +//! job channel and get answers on one-shot channels, so they never block on +//! CUDA. The helper may block: it synchronizes the request's stream after each +//! page before it answers, which is what makes a page visible to the client. +//! +//! Each request gets its own stream and import. Jobs for one request run in +//! order, and a request has at most one job in flight, because the client path +//! awaits each page before the next. [`MAX_OPEN`] bounds requests. +//! +//! When a request ends without [`DirectSink::finish`] (for example the client +//! disconnected), dropping its sink queues a release job behind any page copy +//! still running. The helper releases the stream first, which synchronizes it, +//! then the import. `racer_gpu` leaks and counts anything whose release fails, +//! so memory a GPU may still write is never freed. +//! +//! # Sealed pages +//! +//! Pages that are not verified plaintext in memory arrive sealed +//! ([`crate::read::sealed::SealedPage`]) and the GPU authenticates and decrypts +//! them. Each GPU has one staging buffer of [`SLOT_BYTES`] slots, allocated on +//! first use and capped by `RACER_GPU_STAGING_BYTES`: +//! +//! - Slot 0 holds host ciphertext copied in for the current job. +//! - Slot 1 holds plaintext when a range covers only part of a page: the GPU +//! decrypts the whole page there (the tag covers all of it), checks the +//! verdict, then copies only the requested bytes into the import. +//! - The other slots hold ciphertext that a page's owning worker read from a +//! local v2 disk record with cuFile ([`GpuHelper::stage_file`]). A +//! [`StagedCiphertext`] owns its slot and returns it on drop. When every +//! slot is taken, the owner reads the record through host memory instead. +//! +//! The helper runs one job at a time and synchronizes before it answers, so the +//! two scratch slots are never shared by two jobs and no staged slot is in use +//! by a stream between jobs, which [`racer_gpu::GpuFile::read`] requires. +//! +//! # RDMA landing +//! +//! When RDMA is active, each rail may get one GPU's landing buffer +//! ([`GpuHelper::landing`]): separate GPU memory, capped by +//! `RACER_GPU_LANDING_BYTES`, exported once as a dma-buf and registered with +//! the rail's NIC as an `rdma_verbs` device pool. A peer's NIC writes a +//! `gpu-read` page's ciphertext straight into it, and the GPU decrypts from +//! there ([`LandedCiphertext`]). The CPU never reads or copies those bytes. +//! Landing buffers are never freed while the process runs, and are leaked on +//! purpose when the helper exits, because a NIC registration may still cover +//! them. +//! +//! At startup the helper probes each GPU with a zero-length decrypt. A GPU +//! whose shim has no decrypt kernels, or whose staging cap cannot hold the two +//! scratch slots, gets plaintext pages only: the CPU decrypts as for a Get. + +use crate::client::direct::Destination; +use crate::client::direct::DirectPage; +use crate::client::direct::DirectSink; +use crate::client::direct::GpuService; +use crate::config::GpuMode; +use crate::error::Error; +use crate::error::Operation; +use crate::error::Result; +use crate::memory::VerifiedPage; +use crate::model::PAGE_BYTES; +use crate::read::sealed::SealedPage; +use crate::read::sealed::SealedSource; +use crate::read::sealed::SealedTarget; +use crate::runtime::RequestScope; +use futures::channel::oneshot; +use racer_gpu::DeviceInfo; +use std::collections::HashMap; +use std::ops::Range; +use std::os::fd::OwnedFd; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::sync::mpsc; +use std::task::Poll; + +/// Concurrent `gpu-read` requests per process. More get 503. +pub const MAX_OPEN: usize = 256; + +/// Bytes per staging slot: one sealed page (plaintext and tag), rounded up +/// to whole 4 KiB blocks so cuFile reads land block aligned. +pub const SLOT_BYTES: u64 = (PAGE_BYTES + 16).next_multiple_of(4096); + +/// Scratch slots each job may use: host ciphertext and partial plaintext. +const SCRATCH_SLOTS: u64 = 2; + +/// Offset of the host ciphertext scratch slot. +const CIPHERTEXT_SCRATCH: u64 = 0; + +/// Offset of the partial plaintext scratch slot. +const PLAINTEXT_SCRATCH: u64 = SLOT_BYTES; + +/// Registered cuFile handles kept per process; more are reopened on demand. +const MAX_FILES: usize = 64; + +/// Threads that run cuFile reads. One synchronous read keeps an NVMe drive +/// far from busy, and reads on the helper thread would stall decryption. +const FILE_READERS: usize = 4; + +/// A shared ciphertext page the GPU stream holds until its copy completes. +struct HostCiphertext(crate::memory::CiphertextPage); + +impl AsRef<[u8]> for HostCiphertext { + fn as_ref(&self) -> &[u8] { + self.0.bytes() + } +} + +/// Builds the GPU backend on the helper thread. +pub type Loader = Box racer_gpu::Result + Send>; + +/// Plaintext bytes handed to the helper. The page is immutable and its +/// storage is reference counted, so the helper reads it in place; the handle +/// goes back with the answer so the page is released on the I/O thread. +#[derive(Clone)] +struct Plaintext { + page: VerifiedPage, + range: Range, +} + +impl AsRef<[u8]> for Plaintext { + fn as_ref(&self) -> &[u8] { + self.bytes() + } +} + +impl Plaintext { + fn bytes(&self) -> &[u8] { + &self.page.bytes()[self.range.clone()] + } +} + +/// A sealed page and the bytes of its plaintext the request wants. +struct Sealed { + page: Box, + range: Range, +} + +/// Work for one request on the GPU. +enum Work { + /// Copy verified plaintext from host memory into the import. + Plaintext(Plaintext), + + /// Verify and decrypt a sealed page into the import. + Sealed(Sealed), +} + +/// The staged-ciphertext slots of one GPU. Shared with every +/// [`StagedCiphertext`] so a slot comes back when its page is dropped, on any +/// thread, without a message to the helper. +struct SlotPool { + free: Mutex>, +} + +impl SlotPool { + fn take(&self) -> Option { + self.free.lock().ok()?.pop() + } + + fn put(&self, slot: u64) { + if let Ok(mut free) = self.free.lock() { + free.push(slot); + } + } + + fn available(&self) -> usize { + self.free.lock().map_or(0, |free| free.len()) + } +} + +/// Ciphertext in one staging slot of one GPU, read there by cuFile. Owns the +/// slot until dropped. Only the helper thread touches the bytes. +pub struct StagedCiphertext { + pool: Arc, + ordinal: i32, + slot: u64, +} + +impl StagedCiphertext { + /// The GPU the bytes are on. + pub fn ordinal(&self) -> i32 { + self.ordinal + } + + /// Byte offset of the slot in the GPU's staging buffer. + fn offset(&self) -> u64 { + self.slot * SLOT_BYTES + } +} + +impl Drop for StagedCiphertext { + fn drop(&mut self) { + self.pool.put(self.slot); + } +} + +/// Each GPU's landing buffer gets its own NIC address range from here. Any +/// value works: the address is local to the memory registration. +const LANDING_IOVA: u64 = 1 << 40; + +/// One GPU's RDMA landing buffer, shared with `rdma_verbs` as a dma-buf. +/// See the module docs, "RDMA landing". +pub struct GpuLanding { + /// Export of the whole landing buffer. + fd: OwnedFd, + ordinal: i32, + length: u64, +} + +impl GpuLanding { + /// The GPU the memory is on. + pub fn ordinal(&self) -> i32 { + self.ordinal + } +} + +impl rdma_verbs::DmabufMemory for GpuLanding { + fn fd(&self) -> std::os::fd::BorrowedFd<'_> { + use std::os::fd::AsFd; + self.fd.as_fd() + } + + fn offset(&self) -> u64 { + 0 + } + + fn length(&self) -> usize { + self.length as usize + } + + fn iova(&self) -> u64 { + LANDING_IOVA * (self.ordinal as u64 + 1) + } +} + +/// Peer ciphertext that a NIC wrote into a GPU landing buffer. Holds the +/// RDMA device page, so the slot is not reused until this is dropped. +pub struct LandedCiphertext { + /// Holds the slot. `None` only for test pages written without a NIC. + _page: Option, + ordinal: i32, + offset: u64, + length: u64, +} + +impl LandedCiphertext { + /// Wrap `page` if it lies in `landing`'s memory. + pub fn new(landing: &Arc, page: rdma_verbs::DevicePage) -> Result { + let memory = Arc::as_ptr(page.memory()).cast::<()>(); + if !std::ptr::eq(memory, Arc::as_ptr(landing).cast::<()>()) { + return Err(Error::Internal); + } + Ok(Self { + ordinal: landing.ordinal, + offset: page.offset(), + length: page.len() as u64, + _page: Some(page), + }) + } + + /// Bytes at `offset` of `landing`, written there by a test in place of + /// a NIC. + #[cfg(test)] + pub(crate) fn for_test(landing: &GpuLanding, offset: u64, length: u64) -> Self { + assert!(offset + length <= landing.length); + Self { + _page: None, + ordinal: landing.ordinal, + offset, + length, + } + } + + /// The GPU the bytes are on. + pub fn ordinal(&self) -> i32 { + self.ordinal + } + + /// Bytes that landed. + pub fn len(&self) -> u64 { + self.length + } + + /// Never true: landed ciphertext always holds a tag. + pub fn is_empty(&self) -> bool { + self.length == 0 + } + + /// Byte offset in the landing buffer. + pub(crate) fn offset(&self) -> u64 { + self.offset + } +} + +/// Counters for GPU decryption, for logs and tests. +#[derive(Debug, Default)] +pub struct GpuCounters { + /// Sealed pages the GPU verified and wrote. + pub decrypted: AtomicU64, + + /// Sealed pages the GPU rejected (CRC or tag mismatch). + pub rejected: AtomicU64, + + /// Pages cuFile read straight into staging. + pub staged: AtomicU64, + + /// Direct reads that found no free staging slot. + pub exhausted: AtomicU64, + + /// cuFile reads or registrations that failed. + pub file_failures: AtomicU64, + + /// Sealed pages decrypted from an RDMA landing buffer. + pub landed: AtomicU64, +} + +/// Per-GPU state shared with I/O threads. +struct DeviceShared { + ordinal: i32, + + /// The startup decrypt probe passed and staging holds the scratch slots. + decrypt: bool, + + slots: Arc, +} + +/// A message to the helper thread. +enum Job { + Import { + id: u64, + fd: OwnedFd, + allocation_size: u64, + window: Range, + reply: oneshot::Sender>, + }, + /// Read ciphertext from a file straight into a staging slot. + Stage { + staged: StagedCiphertext, + fd: OwnedFd, + file_offset: u64, + length: u64, + reply: oneshot::Sender>, + }, + Write { + id: u64, + offset: u64, + work: Work, + reply: oneshot::Sender<(Work, Result<()>)>, + }, + Release { + id: u64, + reply: Option>>, + }, + /// Allocate a GPU's RDMA landing buffer and export it as a dma-buf. + Landing { + ordinal: i32, + length: u64, + reply: oneshot::Sender>, + }, + #[cfg(test)] + Run(Box), +} + +/// Handle to the GPU helper thread. Shared by every worker. +pub struct GpuHelper { + jobs: mpsc::Sender, + devices: Vec, + next: AtomicU64, + open: AtomicUsize, + leaked: Arc, + shared: Arc>, + /// cuFile is worth trying: no driver-level failure has been seen. + cufile: Arc, + counters: Arc, + kernels_off: Arc>>, + /// Landing buffers made so far, one per GPU at most. + landings: Mutex>>, +} + +impl GpuHelper { + /// Resolve `mode` against hardware. Returns `None` when GPU mode is off. + pub fn start(mode: GpuMode, staging_bytes: u64) -> Result>> { + match mode { + GpuMode::Disabled => Ok(None), + GpuMode::Auto if !cfg!(feature = "cuda") => { + eprintln!("racer-dataplane: stage=gpu state=disabled reason=not-built"); + Ok(None) + } + _ => Self::start_with_staging(mode, staging_bytes, Box::new(racer_gpu::Gpu::load)), + } + } + + /// Start the helper with a custom backend loader and the default staging cap. + pub fn start_with(mode: GpuMode, loader: Loader) -> Result>> { + Self::start_with_staging(mode, crate::config::DEFAULT_GPU_STAGING_BYTES, loader) + } + + /// Start the helper with a custom backend loader and a per-GPU staging cap. + pub fn start_with_staging( + mode: GpuMode, + staging_bytes: u64, + loader: Loader, + ) -> Result>> { + if mode == GpuMode::Disabled { + return Ok(None); + } + let (jobs, receiver) = mpsc::channel(); + let (ready, started) = mpsc::sync_channel(1); + let leaked = Arc::new(AtomicU64::new(0)); + let cufile = Arc::new(AtomicBool::new(true)); + let counters = Arc::new(GpuCounters::default()); + let slots = staging_bytes / SLOT_BYTES; + let kernels_off = Arc::new(Mutex::new(std::collections::HashSet::new())); + let state = HelperState { + leaked: leaked.clone(), + cufile: cufile.clone(), + counters: counters.clone(), + kernels_off: kernels_off.clone(), + slots, + }; + std::thread::Builder::new() + .name("racer-gpu".into()) + .spawn(move || helper_main(loader, receiver, ready, state)) + .map_err(|_| Error::Unavailable)?; + let found = started.recv().map_err(|_| Error::Unavailable)?; + let (devices, decrypt) = match found { + Ok((devices, decrypt)) if !devices.is_empty() => (devices, decrypt), + outcome => { + let reason = match outcome { + Err(error) => error.to_string(), + Ok(_) => "no-usable-gpu".into(), + }; + eprintln!("racer-dataplane: stage=gpu state=disabled reason={reason}"); + return if mode == GpuMode::Required { + Err(Error::Unavailable) + } else { + Ok(None) + }; + } + }; + // Two scratch slots per job; the rest hold cuFile reads. + let decrypting = slots >= SCRATCH_SLOTS; + let shared = devices + .iter() + .zip(&decrypt) + .map(|(device, kernels)| DeviceShared { + ordinal: device.ordinal, + decrypt: *kernels && decrypting, + slots: Arc::new(SlotPool { + free: Mutex::new((SCRATCH_SLOTS..slots.max(SCRATCH_SLOTS)).rev().collect()), + }), + }) + .collect::>(); + eprintln!( + "racer-dataplane: stage=gpu state=enabled usable={} decrypt={} staging_slots={}", + devices.len(), + shared.iter().filter(|device| device.decrypt).count(), + slots, + ); + Ok(Some(Arc::new(Self { + jobs, + devices, + next: AtomicU64::new(1), + open: AtomicUsize::new(0), + leaked, + shared: Arc::new(shared), + cufile, + counters, + kernels_off, + landings: Mutex::new(HashMap::new()), + }))) + } + + /// GPU decryption counters. + pub fn counters(&self) -> &GpuCounters { + &self.counters + } + + fn device(&self, ordinal: i32) -> Option<&DeviceShared> { + self.shared.iter().find(|device| device.ordinal == ordinal) + } + + /// Whether sealed pages can be decrypted on this GPU. + pub fn decrypts(&self, ordinal: i32) -> bool { + self.device(ordinal).is_some_and(|device| device.decrypt) + && self + .kernels_off + .lock() + .is_ok_and(|off| !off.contains(&ordinal)) + } + + /// Free staged-ciphertext slots on this GPU. + pub fn free_slots(&self, ordinal: i32) -> usize { + self.device(ordinal) + .map_or(0, |device| device.slots.available()) + } + + /// Whether a cuFile read is worth trying. + pub fn cufile(&self) -> bool { + self.cufile.load(Ordering::Relaxed) + } + + /// Read `length` bytes at `file_offset` of `fd` into a staging slot on + /// GPU `ordinal` with cuFile. Returns [`Error::Overloaded`] when every + /// slot is taken, and [`Error::Unavailable`] when cuFile cannot read the + /// file; the caller then reads through host memory. The caller must keep + /// the bytes from changing (for example by holding a segment lease) until + /// this resolves. + pub(crate) async fn stage_file( + &self, + ordinal: i32, + fd: OwnedFd, + file_offset: u64, + length: u64, + scope: &RequestScope, + ) -> Result { + if !self.cufile() || length == 0 || length > SLOT_BYTES { + return Err(Error::Unavailable); + } + let device = self.device(ordinal).ok_or(Error::Unavailable)?; + let Some(slot) = device.slots.take() else { + self.counters.exhausted.fetch_add(1, Ordering::Relaxed); + return Err(Error::Overloaded); + }; + let staged = StagedCiphertext { + pool: device.slots.clone(), + ordinal, + slot, + }; + let (reply, mut receiver) = oneshot::channel(); + self.send(Job::Stage { + staged, + fd, + file_offset, + length, + reply, + })?; + // On cancellation the helper still runs the read; the staged slot + // then drops with the unread answer and returns to the pool. + answer(&mut receiver, scope).await? + } + + /// GPU `ordinal`'s RDMA landing buffer of `length` bytes (a multiple of + /// 4 KiB), allocated and exported on first call and reused after that. + /// [`Error::Unavailable`] if the GPU does not decrypt or the export + /// fails, [`Error::Overloaded`] if the GPU has no room. + pub(crate) async fn landing( + &self, + ordinal: i32, + length: u64, + scope: &RequestScope, + ) -> Result> { + if !self.decrypts(ordinal) || length == 0 || !length.is_multiple_of(4096) { + return Err(Error::Unavailable); + } + if let Some(landing) = self + .landings + .lock() + .ok() + .and_then(|l| l.get(&ordinal).cloned()) + { + return Ok(landing); + } + let (reply, mut receiver) = oneshot::channel(); + self.send(Job::Landing { + ordinal, + length, + reply, + })?; + let fd = answer(&mut receiver, scope).await??; + let mut landings = self.landings.lock().map_err(|_| Error::Internal)?; + Ok(landings + .entry(ordinal) + .or_insert_with(|| { + Arc::new(GpuLanding { + fd, + ordinal, + length, + }) + }) + .clone()) + } + + /// Where a request on GPU `ordinal` should have sealed pages delivered, + /// or None to get plaintext only. + pub fn sealed_target(self: &Arc, ordinal: i32) -> Option { + self.decrypts(ordinal).then(|| SealedTarget { + helper: self.clone(), + ordinal, + }) + } + + /// Usable GPUs found at startup. + pub fn devices(&self) -> &[DeviceInfo] { + &self.devices + } + + /// GPU resources leaked on purpose after a failed release. + pub fn leaked(&self) -> u64 { + self.leaked.load(Ordering::Relaxed) + } + + /// Requests that currently hold a sink. + pub fn open_requests(&self) -> usize { + self.open.load(Ordering::Relaxed) + } + + fn send(&self, job: Job) -> Result<()> { + self.jobs.send(job).map_err(|_| Error::Unavailable) + } + + /// Run `f` on the helper thread and wait for it. Tests use this to reach + /// thread-local simulation state. + #[cfg(test)] + pub(crate) fn run(&self, f: impl FnOnce() -> R + Send + 'static) -> R { + let (tx, rx) = mpsc::channel(); + self.send(Job::Run(Box::new(move || { + let _ = tx.send(f()); + }))) + .unwrap(); + rx.recv().unwrap() + } +} + +/// The per-worker [`GpuService`]. +pub struct GpuClient { + helper: Arc, +} + +impl GpuClient { + pub fn new(helper: Arc) -> Self { + Self { helper } + } +} + +impl GpuService for GpuClient { + fn open( + &self, + descriptor: OwnedFd, + destination: Destination, + allocation_size: u64, + ) -> Result> { + let window = destination.offset..destination.end(); + if window.start >= window.end || window.end > allocation_size { + return Err(Error::InvalidRequest); + } + let open = &self.helper.open; + if open.fetch_add(1, Ordering::AcqRel) >= MAX_OPEN { + open.fetch_sub(1, Ordering::AcqRel); + return Err(Error::Overloaded); + } + let mut sink = GpuSink { + helper: self.helper.clone(), + id: self.helper.next.fetch_add(1, Ordering::Relaxed), + window: window.clone(), + import: None, + ordinal: None, + released: false, + }; + let (reply, answer) = oneshot::channel(); + sink.helper.send(Job::Import { + id: sink.id, + fd: descriptor, + allocation_size, + window, + reply, + })?; + sink.import = Some(answer); + Ok(Box::new(sink)) + } +} + +/// One `gpu-read` request's view of its import on the helper. +struct GpuSink { + helper: Arc, + id: u64, + window: Range, + /// The pending import answer, until it has been checked. + import: Option>>, + /// The GPU that holds the allocation, once imported. + ordinal: Option, + released: bool, +} + +/// Wait for a helper answer, or stop early when the request is canceled. +/// Abandoning the wait never abandons the job: the helper still runs it. +async fn answer(receiver: &mut oneshot::Receiver, scope: &RequestScope) -> Result { + let registration = scope.cancellation.subscribe()?; + std::future::poll_fn(|cx| { + registration.register(cx.waker()); + if let Poll::Ready(result) = std::pin::Pin::new(&mut *receiver).poll(cx) { + return Poll::Ready(result.map_err(|_| Error::Unavailable)); + } + scope.check()?; + Poll::Pending + }) + .await +} + +use std::future::Future; + +impl GpuSink { + async fn imported(&mut self, scope: &RequestScope) -> Result<()> { + if let Some(receiver) = self.import.as_mut() { + let result = answer(receiver, scope).await?; + self.import = None; + self.ordinal = Some(result?); + } + Ok(()) + } +} + +impl DirectSink for GpuSink { + fn ready<'a>(&'a mut self, scope: &'a RequestScope) -> Operation<'a, ()> { + Box::pin(self.imported(scope)) + } + + fn deliver<'a>( + &'a mut self, + page: DirectPage<'a>, + destination_offset: u64, + scope: &'a RequestScope, + ) -> Operation<'a, ()> { + Box::pin(async move { + let end = destination_offset + .checked_add(page.len()) + .ok_or(Error::InvalidRange)?; + if destination_offset < self.window.start || end > self.window.end { + return Err(Error::InvalidRange); + } + self.imported(scope).await?; + let work = match page { + DirectPage::Plaintext(reader) => { + let slice = reader.slice(); + let start = slice.offset as usize; + Work::Plaintext(Plaintext { + page: reader.page().clone(), + range: start..start + slice.length as usize, + }) + } + DirectPage::Sealed { page, range } => Work::Sealed(Sealed { page, range }), + }; + let (reply, mut receiver) = oneshot::channel(); + self.helper.send(Job::Write { + id: self.id, + offset: destination_offset, + work, + reply, + })?; + let (work, result) = answer(&mut receiver, scope).await?; + drop(work); + result + }) + } + + fn sealed_target(&self) -> Option { + self.helper.sealed_target(self.ordinal?) + } + + fn finish<'a>(&'a mut self, scope: &'a RequestScope) -> Operation<'a, ()> { + Box::pin(async move { + if self.released { + return Ok(()); + } + let (reply, mut receiver) = oneshot::channel(); + self.helper.send(Job::Release { + id: self.id, + reply: Some(reply), + })?; + self.released = true; + answer(&mut receiver, scope).await? + }) + } +} + +impl Drop for GpuSink { + fn drop(&mut self) { + if !self.released { + // Queued behind any copy still running; the helper syncs first. + let _ = self.helper.send(Job::Release { + id: self.id, + reply: None, + }); + } + self.helper.open.fetch_sub(1, Ordering::AcqRel); + } +} + +/// One request's GPU objects. Field order matters: the stream drops first, +/// which synchronizes it, then the import. +struct Request { + stream: racer_gpu::Stream, + import: racer_gpu::ImportedAllocation, +} + +/// Shared handles the helper thread starts with. +struct HelperState { + leaked: Arc, + cufile: Arc, + counters: Arc, + /// GPUs whose decrypt kernels failed after startup. + kernels_off: Arc>>, + /// Staging slots per GPU. + slots: u64, +} + +/// A registered cuFile handle and the reads using it. +struct OpenFile { + file: racer_gpu::GpuFile, + /// Reads queued or running on a reader thread. + reads: usize, + /// No longer in [`Helper::files`]: free it once `reads` is zero. + retired: bool, +} + +/// A cuFile read for a reader thread. +struct FileJob { + read: racer_gpu::FileRead, + file: u64, + staged: StagedCiphertext, + reply: oneshot::Sender>, +} + +/// A finished cuFile read. +struct FileDone { + file: u64, + failed: bool, +} + +/// Run cuFile reads until the helper stops sending them. Reports each read +/// to the helper before answering, so a later stage sees a failed handle gone. +fn reader_main( + reader: racer_gpu::FileReader, + jobs: Arc>>, + done: mpsc::Sender, + counters: Arc, +) { + loop { + let job = match jobs.lock() { + Ok(jobs) => jobs.recv(), + Err(_) => return, + }; + let Ok(job) = job else { + return; + }; + let result = reader.read(job.read); + let failed = result.is_err(); + if let Err(error) = &result { + counters.file_failures.fetch_add(1, Ordering::Relaxed); + eprintln!("racer-dataplane: stage=gpu state=cufile-read-failed reason={error}"); + } else { + counters.staged.fetch_add(1, Ordering::Relaxed); + } + let _ = done.send(FileDone { + file: job.file, + failed, + }); + // A dropped receiver drops the slot, which frees it. + let _ = job + .reply + .send(result.map(|()| job.staged).map_err(|_| Error::Unavailable)); + } +} + +/// State owned by the helper thread. +struct Helper { + gpu: racer_gpu::Gpu, + contexts: Vec, + /// Each GPU's staging buffer, allocated on first sealed page. + staging: HashMap, + /// Registered files by (device, inode). The slab descriptors are few and + /// long lived, so one registration serves many reads. + files: HashMap<(u64, u64), u64>, + /// Registered files by id, including retired ones that reads still use. + open: HashMap, + next_file: u64, + /// Reads for the reader threads. `None` once they are told to stop. + reads: Option>, + /// Reads the reader threads finished, so their files can be released. + done: mpsc::Receiver, + readers: Vec>, + /// Opens the cuFile driver ahead of the first sealed page. + opener: Option>, + requests: HashMap, + /// RDMA landing buffers by GPU. Never freed: a NIC may hold a + /// registration over them. + landing: HashMap, + state: HelperState, +} + +type Started = racer_gpu::Result<(Vec, Vec)>; + +fn helper_main( + loader: Loader, + jobs: mpsc::Receiver, + ready: mpsc::SyncSender, + state: HelperState, +) { + let mut helper = match discover(loader) { + Ok((gpu, contexts, usable)) => { + let decrypt = contexts.iter().map(probe_decrypt).collect(); + let (reads, queue) = mpsc::channel(); + let queue = Arc::new(Mutex::new(queue)); + let (finished, done) = mpsc::channel(); + let mut readers = Vec::new(); + for _ in 0..FILE_READERS { + let (reader, queue, finished, counters) = ( + gpu.file_reader(), + queue.clone(), + finished.clone(), + state.counters.clone(), + ); + // Without readers, stages fail and pages go through the host. + if let Ok(thread) = std::thread::Builder::new() + .name("racer-gpu-read".into()) + .spawn(move || reader_main(reader, queue, finished, counters)) + { + readers.push(thread); + } + } + let opener = open_cufile(&gpu, &state.cufile); + let _ = ready.send(Ok((usable, decrypt))); + Helper { + gpu, + contexts, + staging: HashMap::new(), + files: HashMap::new(), + open: HashMap::new(), + next_file: 0, + reads: (!readers.is_empty()).then_some(reads), + done, + readers, + opener, + requests: HashMap::new(), + landing: HashMap::new(), + state, + } + } + Err(error) => { + let _ = ready.send(Err(error)); + return; + } + }; + drop(ready); + while let Ok(job) = jobs.recv() { + helper.finish_reads(); + helper.run(job); + } + // Every sender is gone: let queued reads finish, then release their files. + helper.reads = None; + for reader in std::mem::take(&mut helper.readers) { + let _ = reader.join(); + } + if let Some(opener) = helper.opener.take() { + let _ = opener.join(); + } + helper.finish_reads(); + // Release what is left, stream first. + let ids: Vec = helper.requests.keys().copied().collect(); + for id in ids { + let _ = helper.release(id); + } + // A NIC registration may still cover landing memory: never free it. + for (_, buffer) in helper.landing.drain() { + std::mem::forget(buffer); + } +} + +/// Open the cuFile driver on its own thread. Opening probes every GPU and can +/// take most of a second, so doing it at startup keeps that time out of the +/// first sealed page. A failed open turns cuFile off, as a failed stage does. +fn open_cufile( + gpu: &racer_gpu::Gpu, + cufile: &Arc, +) -> Option> { + if !cufile.load(Ordering::Relaxed) { + return None; + } + let (reader, cufile) = (gpu.file_reader(), cufile.clone()); + std::thread::Builder::new() + .name("racer-gpu-cufile".into()) + .spawn(move || { + if let Err(error) = reader.open_driver() + && cufile.swap(false, Ordering::Relaxed) + { + eprintln!("racer-dataplane: stage=gpu state=cufile-disabled reason={error}"); + } + }) + .ok() +} + +/// Queue a zero-length decrypt to learn whether the shim has working decrypt +/// kernels for this GPU. It writes nothing. +fn probe_decrypt(context: &racer_gpu::Context) -> bool { + let (Ok(staging), Ok(stream)) = (context.staging(4096), context.stream()) else { + return false; + }; + let request = racer_gpu::DecryptRequest { + key: &[0; 32], + nonce: &[0; 12], + aad: &[], + length: 0, + expected_crc: None, + }; + let queued = stream.decrypt( + &request, + &staging, + 0, + racer_gpu::Destination::Staging(&staging), + 64, + ); + let synchronized = stream.synchronize(); + let verdict = synchronized.and_then(|()| stream.verdict()); + let usable = queued.is_ok() && verdict.is_ok_and(|v| v != racer_gpu::Verdict::NotRun); + if !usable { + eprintln!( + "racer-dataplane: stage=gpu ordinal={} decrypt=unavailable reason={:?}", + context.ordinal(), + queued.err() + ); + } + usable +} + +type Discovered = (racer_gpu::Gpu, Vec, Vec); + +/// Load the backend, log every GPU, and retain a context on each usable one. +fn discover(loader: Loader) -> racer_gpu::Result { + let gpu = loader()?; + let mut contexts = Vec::new(); + let mut usable = Vec::new(); + for device in gpu.devices()? { + eprintln!( + "racer-dataplane: stage=gpu ordinal={} uuid={} pci_bus_id={} numa_node={} compute_mode={:?} usable={}", + device.ordinal, + device.uuid_string(), + device.pci_bus_id, + device.numa_node.map_or("unknown".into(), |n| n.to_string()), + device.compute_mode, + device.usable() + ); + if !device.usable() { + continue; + } + match gpu.context(device.ordinal) { + Ok(context) => { + contexts.push(context); + usable.push(device); + } + Err(error) => eprintln!( + "racer-dataplane: stage=gpu ordinal={} state=skipped reason={error}", + device.ordinal + ), + } + } + Ok((gpu, contexts, usable)) +} + +/// Map a GPU error on a write to a request error. +fn gpu_error(error: racer_gpu::Error) -> Error { + match error { + racer_gpu::Error::InvalidRange => Error::InvalidRange, + _ => Error::Unavailable, + } +} + +/// The (device, inode) pair that names an open file. +fn file_identity(fd: &OwnedFd) -> Result<(u64, u64)> { + use std::os::fd::AsRawFd; + // SAFETY: fstat writes only into the zeroed stat buffer. + let mut stat: libc::stat = unsafe { std::mem::zeroed() }; + if unsafe { libc::fstat(fd.as_raw_fd(), &mut stat) } != 0 { + return Err(Error::Unavailable); + } + Ok((stat.st_dev, stat.st_ino)) +} + +/// Map a GPU error on import to an HTTP-visible error. +fn import_error(error: racer_gpu::Error) -> Error { + match error { + racer_gpu::Error::WrongDevice => Error::NotImplemented, + racer_gpu::Error::InvalidRequest + | racer_gpu::Error::InvalidRange + | racer_gpu::Error::Driver(_) => Error::InvalidRequest, + racer_gpu::Error::OutOfMemory | racer_gpu::Error::Busy => Error::Overloaded, + _ => Error::Unavailable, + } +} + +impl Helper { + fn run(&mut self, job: Job) { + match job { + Job::Import { + id, + fd, + allocation_size, + window, + reply, + } => { + let result = self.import(id, fd, allocation_size, window); + self.note_leaks(); + let _ = reply.send(result); + } + Job::Stage { + staged, + fd, + file_offset, + length, + reply, + } => { + let result = self.stage(staged, fd, file_offset, length, reply); + self.note_leaks(); + // A dropped receiver drops the slot, which frees it. + if let Err((error, reply)) = result { + let _ = reply.send(Err(error)); + } + } + Job::Write { + id, + offset, + work, + reply, + } => { + let result = self.write(id, offset, &work); + self.note_leaks(); + // If the I/O side stopped waiting, the page drops here. Its + // storage is shared and thread safe, so that is fine. + let _ = reply.send((work, result)); + } + Job::Release { id, reply } => { + let result = self.release(id); + if let Some(reply) = reply { + let _ = reply.send(result); + } + } + Job::Landing { + ordinal, + length, + reply, + } => { + let result = self.landing(ordinal, length); + self.note_leaks(); + let _ = reply.send(result); + } + #[cfg(test)] + Job::Run(f) => f(), + } + } + + /// Allocate GPU `ordinal`'s landing buffer, once, and export it. + fn landing(&mut self, ordinal: i32, length: u64) -> Result { + if !self.landing.contains_key(&ordinal) { + let context = self + .contexts + .iter() + .find(|context| context.ordinal() == ordinal) + .ok_or(Error::Unavailable)?; + let buffer = context.staging(length).map_err(|error| { + eprintln!( + "racer-dataplane: stage=gpu ordinal={ordinal} state=landing-failed reason={error}" + ); + Error::Overloaded + })?; + self.landing.insert(ordinal, buffer); + } + let buffer = self.landing.get(&ordinal).ok_or(Error::Internal)?; + if buffer.size() != length { + return Err(Error::InvalidConfiguration); + } + buffer.dma_buf(0, length).map_err(|error| { + eprintln!( + "racer-dataplane: stage=gpu ordinal={ordinal} state=landing-export-failed reason={error}" + ); + Error::Unavailable + }) + } + + /// Try each usable GPU until one owns the allocation. + fn import(&mut self, id: u64, fd: OwnedFd, size: u64, window: Range) -> Result { + let mut last = racer_gpu::Error::WrongDevice; + for context in &self.contexts { + let attempt = fd.try_clone().map_err(|_| Error::Unavailable)?; + match context.import(attempt, size, window.clone()) { + Ok(import) => { + let stream = context.stream().map_err(import_error)?; + self.requests.insert(id, Request { stream, import }); + return Ok(context.ordinal()); + } + Err(racer_gpu::Error::WrongDevice) => continue, + Err(error) => { + last = error; + break; + } + } + } + Err(import_error(last)) + } + + fn write(&mut self, id: u64, offset: u64, work: &Work) -> Result<()> { + let plaintext = match work { + Work::Plaintext(plaintext) => plaintext, + Work::Sealed(sealed) => return self.write_sealed(id, offset, sealed), + }; + let request = self.requests.get(&id).ok_or(Error::Internal)?; + let destination = racer_gpu::Destination::Import(&request.import); + let queued = request + .stream + .copy_from_host(destination, offset, plaintext.clone()); + // Synchronize even after a failed queue call: part of it may run. + let synchronized = request.stream.synchronize(); + queued.and(synchronized).map_err(gpu_error) + } + + /// This GPU's staging buffer, allocated on first use. + fn staging(&mut self, ordinal: i32) -> Result<&racer_gpu::StagingBuffer> { + if !self.staging.contains_key(&ordinal) { + let context = self + .contexts + .iter() + .find(|context| context.ordinal() == ordinal) + .ok_or(Error::Internal)?; + let size = self.state.slots.max(SCRATCH_SLOTS) * SLOT_BYTES; + let buffer = context.staging(size).map_err(|error| { + eprintln!( + "racer-dataplane: stage=gpu ordinal={ordinal} state=staging-failed reason={error}" + ); + Error::Overloaded + })?; + self.staging.insert(ordinal, buffer); + } + self.staging.get(&ordinal).ok_or(Error::Internal) + } + + /// Queue a cuFile read of a file range into a staged slot. A reader + /// thread answers `reply`; on an early error it comes back to the caller. + fn stage( + &mut self, + staged: StagedCiphertext, + fd: OwnedFd, + file_offset: u64, + length: u64, + reply: oneshot::Sender>, + ) -> std::result::Result<(), (Error, oneshot::Sender>)> { + let Some(reads) = self.reads.clone() else { + return Err((Error::Unavailable, reply)); + }; + let identity = match file_identity(&fd) { + Ok(identity) => identity, + Err(error) => return Err((error, reply)), + }; + if let Err(error) = self.staging(staged.ordinal) { + return Err((error, reply)); + } + let Some(staging) = self.staging.get(&staged.ordinal) else { + return Err((Error::Internal, reply)); + }; + if let Err(error) = staging.register_cufile() { + self.state + .counters + .file_failures + .fetch_add(1, Ordering::Relaxed); + self.disable_cufile(error); + return Err((Error::Unavailable, reply)); + } + let id = match self.files.get(&identity) { + Some(id) => *id, + None => { + if self.files.len() >= MAX_FILES { + for id in std::mem::take(&mut self.files).into_values() { + self.retire(id); + } + } + match self.gpu.open_file(fd) { + Ok(file) => { + let id = self.next_file; + self.next_file += 1; + self.open.insert( + id, + OpenFile { + file, + reads: 0, + retired: false, + }, + ); + self.files.insert(identity, id); + id + } + Err(error) => { + self.state + .counters + .file_failures + .fetch_add(1, Ordering::Relaxed); + self.disable_cufile(error); + return Err((Error::Unavailable, reply)); + } + } + } + }; + let (Some(open), Some(staging)) = + (self.open.get_mut(&id), self.staging.get(&staged.ordinal)) + else { + return Err((Error::Internal, reply)); + }; + // SAFETY: the file stays in `open` while `reads` is above zero, the + // staging buffer and backend live until the readers are joined, and + // the slot is this job's alone until the read answers. + let read = match unsafe { + open.file + .detach(staging, staged.offset(), file_offset, length) + } { + Ok(read) => read, + Err(error) => return Err((gpu_error(error), reply)), + }; + open.reads += 1; + let job = FileJob { + read, + file: id, + staged, + reply, + }; + if let Err(mpsc::SendError(job)) = reads.send(job) { + open.reads -= 1; + return Err((Error::Unavailable, job.reply)); + } + Ok(()) + } + + /// Account for reads the reader threads finished. A failed handle may be + /// stale, so it is registered again next time. + fn finish_reads(&mut self) { + while let Ok(FileDone { file, failed }) = self.done.try_recv() { + if let Some(open) = self.open.get_mut(&file) { + open.reads = open.reads.saturating_sub(1); + } + if failed { + self.files.retain(|_, id| *id != file); + self.retire(file); + } else { + self.release_file(file); + } + } + } + + /// Stop handing out a file; free it once no read uses it. + fn retire(&mut self, id: u64) { + if let Some(open) = self.open.get_mut(&id) { + open.retired = true; + } + self.release_file(id); + } + + /// Free a retired file that no read uses. + fn release_file(&mut self, id: u64) { + if self + .open + .get(&id) + .is_some_and(|open| open.retired && open.reads == 0) + { + self.open.remove(&id); + } + } + + /// Stop trying cuFile after a failure that is not about one read. + fn disable_cufile(&self, error: racer_gpu::Error) { + if self.state.cufile.swap(false, Ordering::Relaxed) { + eprintln!("racer-dataplane: stage=gpu state=cufile-disabled reason={error}"); + } + } + + /// Verify and decrypt a sealed page, then write only the requested range. + fn write_sealed(&mut self, id: u64, offset: u64, sealed: &Sealed) -> Result<()> { + let ordinal = self + .requests + .get(&id) + .ok_or(Error::Internal)? + .import + .ordinal(); + self.staging(ordinal)?; + let staging = self.staging.get(&ordinal).ok_or(Error::Internal)?; + let request = self.requests.get(&id).ok_or(Error::Internal)?; + let page = &sealed.page; + let length = u64::from(page.plaintext_length()); + if sealed.range.start >= sealed.range.end || sealed.range.end > length { + return Err(Error::InvalidRange); + } + let stream = &request.stream; + let (source_buffer, source) = match &page.source { + SealedSource::Host(copy) => { + let bytes = copy.copy.ciphertext.bytes(); + if bytes.len() as u64 != length + 16 { + return Err(Error::CorruptRecord); + } + let copied = stream.copy_from_host( + racer_gpu::Destination::Staging(staging), + CIPHERTEXT_SCRATCH, + HostCiphertext(copy.copy.ciphertext.clone()), + ); + if let Err(error) = copied { + let _ = stream.synchronize(); + return Err(gpu_error(error)); + } + (staging, CIPHERTEXT_SCRATCH) + } + SealedSource::Staged { staged, .. } => { + if staged.ordinal() != ordinal { + return Err(Error::Internal); + } + (staging, staged.offset()) + } + SealedSource::Landed(landed) => { + if landed.ordinal() != ordinal || landed.len() != length + 16 { + return Err(Error::Internal); + } + let buffer = self.landing.get(&ordinal).ok_or(Error::Internal)?; + (buffer, landed.offset()) + } + }; + let request_key = racer_gpu::DecryptRequest { + key: page.key.key(), + nonce: page.key.nonce(), + aad: &page.aad, + length, + expected_crc: page.expected_crc, + }; + // A whole page decrypts straight into the import. Part of a page + // decrypts into scratch first, because the tag covers the whole page + // and nothing outside the window may be written. + let whole = sealed.range.start == 0 && sealed.range.end == length; + let (destination, at) = if whole { + (racer_gpu::Destination::Import(&request.import), offset) + } else { + (racer_gpu::Destination::Staging(staging), PLAINTEXT_SCRATCH) + }; + let queued = stream.decrypt(&request_key, source_buffer, source, destination, at); + let synchronized = stream.synchronize(); + if let Err(racer_gpu::Error::KernelUnavailable) = queued { + if let Ok(mut off) = self.state.kernels_off.lock() { + off.insert(ordinal); + } + return Err(Error::Unavailable); + } + queued.and(synchronized).map_err(gpu_error)?; + match stream.verdict().map_err(gpu_error)? { + racer_gpu::Verdict::Passed { .. } => {} + racer_gpu::Verdict::CrcMismatch { .. } | racer_gpu::Verdict::TagMismatch { .. } => { + self.state.counters.rejected.fetch_add(1, Ordering::Relaxed); + return Err(Error::CorruptRecord); + } + racer_gpu::Verdict::NotRun => return Err(Error::Unavailable), + } + if !whole { + let copied = stream.copy_from_staging( + racer_gpu::Destination::Import(&request.import), + offset, + staging, + PLAINTEXT_SCRATCH + sealed.range.start, + sealed.range.end - sealed.range.start, + ); + let synchronized = stream.synchronize(); + copied.and(synchronized).map_err(gpu_error)?; + } + self.state + .counters + .decrypted + .fetch_add(1, Ordering::Relaxed); + if matches!(page.source, SealedSource::Landed(_)) { + self.state.counters.landed.fetch_add(1, Ordering::Relaxed); + } + Ok(()) + } + + /// Drop the stream (which synchronizes it), then the import. + fn release(&mut self, id: u64) -> Result<()> { + let Some(request) = self.requests.remove(&id) else { + return Ok(()); + }; + let before = self.gpu.leaked(); + let synchronized = request.stream.synchronize(); + let Request { stream, import } = request; + drop(stream); + drop(import); + self.note_leaks(); + if synchronized.is_err() || self.gpu.leaked() != before { + return Err(Error::Unavailable); + } + Ok(()) + } + + fn note_leaks(&self) { + let leaked = self.gpu.leaked(); + if self.state.leaked.swap(leaked, Ordering::Relaxed) != leaked { + eprintln!("racer-dataplane: stage=gpu state=leaked total={leaked}"); + } + } +} + +#[cfg(test)] +pub(crate) mod testing { + //! A GPU helper on simulated GPUs, for tests outside the client socket. + use super::*; + use std::cell::RefCell; + + thread_local! { + /// The simulation, on the GPU helper thread only. + static SIM: RefCell> = + const { RefCell::new(None) }; + } + + /// The helper thread's simulation. Call through [`GpuHelper::run`]. + pub(crate) fn simulation() -> racer_gpu::simulation::Simulation { + SIM.with(|sim| sim.borrow().as_ref().unwrap().0.clone()) + } + + /// A test scope with a few seconds on the real clock. + pub(crate) fn scope() -> RequestScope { + RequestScope::new( + crate::model::RequestId([1; 16]), + std::time::Instant::now() + std::time::Duration::from_secs(5), + ) + .unwrap() + } + + /// GPU `ordinal`'s landing buffer of `length` bytes, blocking. + pub(crate) fn landing( + helper: &GpuHelper, + ordinal: i32, + length: u64, + ) -> Result> { + futures::executor::block_on(helper.landing(ordinal, length, &scope())) + } + + /// Write `bytes` at `at` into a landing buffer through its dma-buf, as + /// a NIC would. + pub(crate) fn nic_write(landing: &GpuLanding, at: u64, bytes: &[u8]) { + use std::os::unix::fs::FileExt; + let file = std::fs::File::from(landing.fd.try_clone().unwrap()); + file.write_all_at(bytes, at).unwrap(); + } + + /// Import a fresh allocation on GPU `ordinal` as request `id`. + pub(crate) fn import(helper: &GpuHelper, id: u64, ordinal: i32) -> OwnedFd { + let size = racer_gpu::simulation::GRANULARITY; + let fd = helper.run(move || simulation().allocate(ordinal, size)); + let (reply, answer) = oneshot::channel(); + helper + .send(Job::Import { + id, + fd: fd.try_clone().unwrap(), + allocation_size: size, + window: 0..size, + reply, + }) + .unwrap(); + assert_eq!(futures::executor::block_on(answer).unwrap(), Ok(ordinal)); + fd + } + + /// Verify and decrypt `page`'s `range` into request `id` at offset 0. + pub(crate) fn write( + helper: &GpuHelper, + id: u64, + page: Box, + range: Range, + ) -> Result<()> { + let (reply, answer) = oneshot::channel(); + helper + .send(Job::Write { + id, + offset: 0, + work: Work::Sealed(Sealed { page, range }), + reply, + }) + .unwrap(); + futures::executor::block_on(answer).unwrap().1 + } + + /// Host-to-device copies the GPU has made so far. + pub(crate) fn host_copies(helper: &GpuHelper) -> usize { + helper.run(|| { + simulation() + .trace() + .iter() + .filter(|event| event.operation == racer_gpu::simulation::Operation::CopyH2d) + .count() + }) + } + + /// A required-mode helper on simulated GPUs with these NUMA nodes. + pub(crate) fn helper(numa: &[Option]) -> Arc { + helper_with(numa, |_| {}) + } + + /// A helper whose simulation `setup` prepares before the backend loads. + pub(crate) fn helper_with( + numa: &[Option], + setup: impl FnOnce(&racer_gpu::simulation::Simulation) + Send + 'static, + ) -> Arc { + let devices: Vec = numa + .iter() + .enumerate() + .map(|(ordinal, node)| { + let mut device = racer_gpu::simulation::SimDevice::new(ordinal as u8); + device.numa_node = *node; + device + }) + .collect(); + GpuHelper::start_with( + GpuMode::Required, + Box::new(move || { + let simulation = racer_gpu::simulation::Simulation::new(devices); + setup(&simulation); + let environment = simulation.enter(); + SIM.with(|sim| *sim.borrow_mut() = Some((simulation, environment))); + racer_gpu::Gpu::load() + }), + ) + .unwrap() + .unwrap() + } +} + +#[cfg(test)] +mod tests { + //! RDMA landing buffers on simulated GPUs: allocation, reuse, and + //! decrypting ciphertext a NIC wrote straight into GPU memory. + use super::testing::*; + use super::*; + use crate::model::*; + use rdma_verbs::DmabufMemory; + + const LENGTH: u64 = 2 * SLOT_BYTES; + + fn metadata(length: u64) -> ObjectMetadata { + ObjectMetadata { + content_type: None, + version: ObjectVersion { + object: ObjectId { + cache: racer_control_wire::CacheId( + "00000000-0000-4000-8000-000000000001".into(), + ), + key: CacheKey([0; 32]), + }, + etag: StrongEtag::test_value("v1"), + }, + length, + expires_at: ExpiresAt::from_system_time(std::time::UNIX_EPOCH).unwrap(), + } + } + + /// A sealed page whose real ciphertext a NIC wrote at `at` in `landing`. + fn landed(landing: &GpuLanding, at: u64, plaintext: &[u8], tamper: bool) -> Box { + let metadata = metadata(plaintext.len() as u64); + let envelope = PageEnvelope { + cipher: PageCipher::Xaes256Gcm, + page: PageId { + version: metadata.version.clone(), + number: PageNumber(0), + }, + key_id: key_id_from_generation(1, 1).unwrap(), + nonce: Nonce([2; 24]), + plaintext_length: plaintext.len() as u32, + ciphertext_length: plaintext.len() as u32 + 16, + }; + let aad = crate::security::page_aad(&envelope).unwrap(); + let mut sealed = vec![0; plaintext.len() + 16]; + racer_crypto::seal(&[7; 32], &envelope.nonce.0, &aad, plaintext, &mut sealed).unwrap(); + if tamper { + sealed[3] ^= 1; + } + nic_write(landing, at, &sealed); + let page = Box::new(SealedPage { + metadata, + aad, + key: racer_crypto::xaes_derive(&[7; 32], &envelope.nonce.0), + expected_crc: None, + source: SealedSource::Landed(LandedCiphertext::for_test( + landing, + at, + sealed.len() as u64, + )), + envelope, + }); + page.validate_for(&page.envelope.page.clone()).unwrap(); + page + } + + #[test] + /// One landing buffer per GPU, made once and reused; each GPU gets its + /// own NIC address range; bad sizes and non-decrypting GPUs are refused. + fn landing_buffers_are_per_gpu_and_reused() { + let helper = helper(&[Some(0), Some(1)]); + let first = landing(&helper, 0, LENGTH).unwrap(); + assert!(Arc::ptr_eq(&first, &landing(&helper, 0, LENGTH).unwrap())); + let second = landing(&helper, 1, LENGTH).unwrap(); + assert_eq!((first.ordinal(), second.ordinal()), (0, 1)); + assert_eq!((first.offset(), first.length()), (0, LENGTH as usize)); + assert_ne!(first.iova(), second.iova()); + assert_eq!(landing(&helper, 2, LENGTH).err(), Some(Error::Unavailable)); + assert_eq!(landing(&helper, 0, 0).err(), Some(Error::Unavailable)); + assert_eq!(landing(&helper, 0, 4095).err(), Some(Error::Unavailable)); + assert_eq!(helper.run(|| simulation().live().staging), 2); + assert_eq!(helper.leaked(), 0); + } + + #[test] + /// Allocation or export failures are reported and can be retried. + fn landing_fails_when_allocation_or_export_fails() { + let helper = helper(&[None]); + helper.run(|| simulation().fault(racer_gpu::simulation::Operation::Staging, -4)); + assert_eq!(landing(&helper, 0, LENGTH).err(), Some(Error::Overloaded)); + helper.run(|| simulation().fault(racer_gpu::simulation::Operation::Dmabuf, -5)); + assert_eq!(landing(&helper, 0, LENGTH).err(), Some(Error::Unavailable)); + assert!(landing(&helper, 0, LENGTH).is_ok()); + } + + #[test] + /// Landed ciphertext decrypts on the GPU with no host-to-device copy of + /// the ciphertext, whole or in part. + fn landed_ciphertext_decrypts_on_the_gpu_without_a_host_copy() { + let helper = helper(&[None]); + let buffer = landing(&helper, 0, LENGTH).unwrap(); + let plaintext: Vec = (0..5000).map(|i| (i % 251) as u8).collect(); + let fd = import(&helper, 1, 0); + let before = host_copies(&helper); + write( + &helper, + 1, + landed(&buffer, SLOT_BYTES, &plaintext, false), + 0..5000, + ) + .unwrap(); + assert_eq!( + racer_gpu::simulation::read_allocation(&fd, 0, 5000), + plaintext + ); + assert_eq!(host_copies(&helper), before, "no host copy"); + assert_eq!(helper.counters().landed.load(Ordering::Relaxed), 1); + let fd = import(&helper, 2, 0); + write(&helper, 2, landed(&buffer, 0, &plaintext, false), 10..20).unwrap(); + assert_eq!( + racer_gpu::simulation::read_allocation(&fd, 0, 10), + plaintext[10..20] + ); + assert_eq!(helper.counters().landed.load(Ordering::Relaxed), 2); + assert_eq!(helper.leaked(), 0); + } + + #[test] + /// A tampered landed page is rejected before anything is written; a page + /// on another GPU's landing buffer is refused. + fn tampered_or_misplaced_landed_ciphertext_is_rejected() { + let helper = helper(&[None, None]); + let near = landing(&helper, 0, LENGTH).unwrap(); + let far = landing(&helper, 1, LENGTH).unwrap(); + let plaintext = vec![5; 100]; + let fd = import(&helper, 1, 0); + assert_eq!( + write(&helper, 1, landed(&near, 0, &plaintext, true), 0..100), + Err(Error::CorruptRecord) + ); + assert_eq!( + racer_gpu::simulation::read_allocation(&fd, 0, 100), + vec![0; 100] + ); + assert_eq!(helper.counters().rejected.load(Ordering::Relaxed), 1); + assert_eq!( + write(&helper, 1, landed(&far, 0, &plaintext, false), 0..100), + Err(Error::Internal) + ); + assert_eq!(helper.counters().landed.load(Ordering::Relaxed), 0); + } + + /// A memfd standing in for a slab file. + fn slab(bytes: &[u8]) -> OwnedFd { + use std::os::unix::fs::FileExt; + let fd = unsafe { libc::memfd_create(c"slab".as_ptr(), libc::MFD_CLOEXEC) }; + assert!(fd >= 0); + let file = unsafe { ::from_raw_fd(fd) }; + file.write_all_at(bytes, 0).unwrap(); + file.into() + } + + /// Stage `length` bytes at `offset` of `fd` on GPU 0, blocking. + fn stage( + helper: &GpuHelper, + fd: &OwnedFd, + offset: u64, + length: u64, + ) -> Result { + futures::executor::block_on(helper.stage_file( + 0, + fd.try_clone().unwrap(), + offset, + length, + &scope(), + )) + } + + fn operations(helper: &GpuHelper, operation: racer_gpu::simulation::Operation) -> usize { + helper.run(move || { + simulation() + .trace() + .iter() + .filter(|event| event.operation == operation) + .count() + }) + } + + #[test] + /// Many stages in flight at once share one registration, each gets its + /// own slot, and the slots come back when the answers drop. + fn concurrent_stages_share_one_file_registration() { + use racer_gpu::simulation::Operation; + let helper = helper(&[None]); + let bytes: Vec = (0..(16 * 4096)).map(|i| (i % 253) as u8).collect(); + let fd = slab(&bytes); + let free = helper.free_slots(0); + let staged = futures::executor::block_on(futures::future::join_all((0..8).map(|i| { + let fd = fd.try_clone().unwrap(); + let scope = scope(); + let helper = helper.clone(); + async move { helper.stage_file(0, fd, i * 4096, 4096, &scope).await } + }))); + let staged: Vec<_> = staged.into_iter().map(Result::unwrap).collect(); + let mut slots: Vec<_> = staged.iter().map(|staged| staged.offset()).collect(); + slots.sort_unstable(); + slots.dedup(); + assert_eq!(slots.len(), 8); + assert_eq!(helper.free_slots(0), free - 8); + assert_eq!(helper.counters().staged.load(Ordering::Relaxed), 8); + assert_eq!(operations(&helper, Operation::FileRead), 8); + assert_eq!(operations(&helper, Operation::File), 1); + drop(staged); + assert_eq!(helper.free_slots(0), free); + assert_eq!(helper.run(|| simulation().live().files), 1); + } + + #[test] + /// A failed read frees its slot and the file is registered again for the + /// next read; cuFile stays on. + fn failed_read_registers_the_file_again() { + use racer_gpu::simulation::Operation; + let helper = helper(&[None]); + let fd = slab(&[7; 8192]); + let free = helper.free_slots(0); + stage(&helper, &fd, 0, 4096).unwrap(); + helper.run(|| simulation().fault(Operation::FileRead, -5)); + assert_eq!(stage(&helper, &fd, 0, 4096).err(), Some(Error::Unavailable)); + assert_eq!(helper.free_slots(0), free); + assert_eq!(helper.counters().file_failures.load(Ordering::Relaxed), 1); + assert!(helper.cufile()); + stage(&helper, &fd, 4096, 4096).unwrap(); + assert_eq!(operations(&helper, Operation::File), 2); + assert_eq!(operations(&helper, Operation::FileFree), 1); + assert_eq!(helper.run(|| simulation().live().files), 1); + assert_eq!(helper.counters().staged.load(Ordering::Relaxed), 2); + } + + /// Wait for the helper's background work to reach `done`. + fn eventually(mut done: impl FnMut() -> bool) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !done() { + assert!(std::time::Instant::now() < deadline, "timed out"); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } + + #[test] + /// The cuFile driver opens at startup, before any sealed page asks for it. + fn cufile_driver_opens_at_startup() { + use racer_gpu::simulation::Operation; + let helper = helper(&[None]); + eventually(|| operations(&helper, Operation::FileOpen) == 1); + assert!(helper.cufile()); + let fd = slab(&[3; 4096]); + stage(&helper, &fd, 0, 4096).unwrap(); + assert_eq!(helper.counters().staged.load(Ordering::Relaxed), 1); + } + + #[test] + /// A driver that fails to open at startup turns cuFile off before any + /// sealed page asks for it. + fn cufile_driver_open_failure_turns_cufile_off() { + use racer_gpu::simulation::Operation; + let helper = helper_with(&[None], |simulation| { + simulation.fault(Operation::FileOpen, -5) + }); + eventually(|| !helper.cufile()); + assert_eq!(helper.counters().staged.load(Ordering::Relaxed), 0); + } + + #[test] + /// A short file read fails cleanly, as it would on a truncated slab. + fn short_read_fails_the_stage() { + let helper = helper(&[None]); + let fd = slab(&[1; 4096]); + assert_eq!( + stage(&helper, &fd, 2048, 4096).err(), + Some(Error::Unavailable) + ); + assert_eq!(helper.counters().file_failures.load(Ordering::Relaxed), 1); + } +} diff --git a/cmd/racer-dataplane/src/http.rs b/cmd/racer-dataplane/src/http.rs index 001b395859..ef536f28c9 100644 --- a/cmd/racer-dataplane/src/http.rs +++ b/cmd/racer-dataplane/src/http.rs @@ -500,6 +500,15 @@ impl ReaderLease { pub fn remaining(&self) -> usize { self.slice.length as usize - self.sent } + /// The verified plaintext of this slice only. + pub fn slice_bytes(&self) -> &[u8] { + let start = self.slice.offset as usize; + &self.page.bytes()[start..start + self.slice.length as usize] + } + /// The verified page this slice pins. + pub fn page(&self) -> &VerifiedPage { + &self.page + } } struct DeliveryOwner(ReaderLease); impl http1::delivery::DirectOwner for DeliveryOwner { @@ -656,6 +665,40 @@ impl Delivery { }) } } +impl Delivery { + /// Write the whole slice into `file` starting at `offset`, with positioned + /// io_uring writes from the shared page. Short writes continue where they + /// stopped; a zero-byte write is an I/O error. Never maps the file. + pub(crate) fn write_slice_at<'a>( + &'a self, + reader: &ReaderLease, + file: Rc, + offset: u64, + scope: &'a RequestScope, + ) -> Operation<'a, ()> { + let page = reader.page.clone(); + let start = reader.slice.offset as usize; + let length = reader.slice.length as usize; + Box::pin(async move { + let mut written = 0; + while written < length { + let view = PageSendRange::new(page.clone(), start + written..start + length)?; + let position = offset + .checked_add(written as u64) + .ok_or(Error::InvalidRange)?; + let completed = self + .reactor + .write_shared_at(file.clone(), position, view, (), scope) + .await?; + if completed.bytes == 0 || completed.bytes > length - written { + return Err(Error::Io); + } + written += completed.bytes; + } + Ok(()) + }) + } +} fn validate_slice(page: &VerifiedPage, slice: PageSlice) -> Result<()> { let end = (slice.offset as usize) .checked_add(slice.length as usize) diff --git a/cmd/racer-dataplane/src/lib.rs b/cmd/racer-dataplane/src/lib.rs index af60905725..4b4014fa9f 100644 --- a/cmd/racer-dataplane/src/lib.rs +++ b/cmd/racer-dataplane/src/lib.rs @@ -12,6 +12,7 @@ pub mod client; pub mod config; pub mod control; pub mod error; +pub mod gpu; pub mod hardware; pub mod http; pub mod memory; diff --git a/cmd/racer-dataplane/src/peer.rs b/cmd/racer-dataplane/src/peer.rs index 3b6fd6da9a..0464c31708 100644 --- a/cmd/racer-dataplane/src/peer.rs +++ b/cmd/racer-dataplane/src/peer.rs @@ -424,6 +424,8 @@ impl Relay { .append_response(response, &previous) .map(transport::RelayResponse::Complete) } + // A relay never asks for landing. + transport::RelayResponse::Landed(_) => Err(Error::Internal), transport::RelayResponse::Http { authentication, connection, @@ -444,6 +446,16 @@ impl Relay { } } +/// What [`Requester::request_landing`] produced. +#[allow(clippy::large_enum_variant)] // Short-lived, returned once per request. +pub(crate) enum Delivered { + /// A verified response, as from [`Requester::request`]. + Response(VerifiedResponse), + + /// A verified page in a GPU landing buffer. + Landed(Box), +} + /// Concrete candidate requester. Scripted outcomes exist only in unit tests and /// the subscription interoperability fixture, so cancellation and an adversarial /// late-success completion fence remain distinct. @@ -483,8 +495,16 @@ pub enum Requester { bool, ) -> Operation<'static, VerifiedResponse>, >, + /// Test-only GPU landing: the landing GPU, and a script that returns + /// a landed page for a request or None to answer it as a plain request. + #[cfg(test)] + landing: Option<(i32, ScriptedLanding)>, }, } + +/// A scripted landed page for one request, or None. +#[cfg(test)] +pub(crate) type ScriptedLanding = Box Option>; impl Requester { #[cfg(any(test, feature = "subscription-interop"))] pub(crate) fn scripted( @@ -520,8 +540,26 @@ impl Requester { } }) }), + #[cfg(test)] + landing: None, }) } + + /// Test-only: let a scripted requester land pages on GPU `ordinal`. + #[cfg(test)] + pub(crate) fn with_scripted_landing( + self: Rc, + ordinal: i32, + land: ScriptedLanding, + ) -> Rc { + let mut requester = Rc::try_unwrap(self) + .ok() + .expect("unshared scripted requester"); + if let Self::Scripted { landing, .. } = &mut requester { + *landing = Some((ordinal, land)); + } + Rc::new(requester) + } pub fn new( paths: Rc, forwarding: Rc, @@ -677,7 +715,15 @@ impl Requester { let (signed, binding) = forwarding.sign_request_to(request, &next)?; let mut timing = PageTiming::new(metrics); let response = self - .exchange_inner_mode(signed, membership, None, &scope, true, Some(&mut timing)) + .exchange_inner_mode( + signed, + membership, + None, + &scope, + true, + Some(&mut timing), + None, + ) .await?; let transport::RelayResponse::Complete(response) = response else { return Err(Error::Internal); @@ -698,6 +744,73 @@ impl Requester { membership: std::sync::Arc, scope: &'a RequestScope, ) -> Operation<'a, VerifiedResponse> { + Box::pin(async move { + match self + .request_network_landing(request, membership, None, scope) + .await? + { + Delivered::Response(response) => Ok(response), + Delivered::Landed(_) => Err(Error::Internal), + } + }) + } + + /// Like [`Self::request`] for one fixed page that a `gpu-read` will + /// decrypt on GPU `ordinal`. If the last hop is RDMA and this node has a + /// free landing slot on that GPU's rail, the page lands in GPU memory and + /// comes back as [`Delivered::Landed`]. Otherwise this is a plain request. + pub(crate) fn request_landing<'a>( + &'a self, + request: PeerRequest, + membership: std::sync::Arc, + ordinal: i32, + scope: &'a RequestScope, + ) -> Operation<'a, Delivered> { + match self { + Self::Network { .. } => { + self.request_network_landing(request, membership, Some(ordinal), scope) + } + #[cfg(any(test, feature = "subscription-interop"))] + Self::Scripted { + request: script, + #[cfg(test)] + landing, + .. + } => { + #[cfg(test)] + if let Some((own, land)) = landing + && *own == ordinal + && let Some(landed) = land(&request) + { + return Box::pin(async move { Ok(Delivered::Landed(Box::new(landed))) }); + } + let response = script(request, membership, scope.clone(), false); + Box::pin(async move { response.await.map(Delivered::Response) }) + } + } + } + + /// Whether this node has a landing buffer for GPU `ordinal` on any rail. + pub(crate) fn landing_ready(&self, ordinal: i32) -> bool { + match self { + Self::Network { transfers, .. } => transfers.landing_ready(ordinal), + #[cfg(test)] + Self::Scripted { + landing: Some((own, _)), + .. + } => *own == ordinal, + #[cfg(any(test, feature = "subscription-interop"))] + Self::Scripted { .. } => false, + } + } + + fn request_network_landing<'a>( + &'a self, + request: PeerRequest, + membership: std::sync::Arc, + landing: Option, + scope: &'a RequestScope, + ) -> Operation<'a, Delivered> { let (network, paths, observer, forwarding, metrics) = match self { Self::Network { network, @@ -727,10 +840,24 @@ impl Requester { let (signed, binding) = forwarding.sign_request_to(request, next)?; let mut timing = PageTiming::new(metrics); let response = self - .exchange_inner_mode(signed, membership, None, &scope, false, Some(&mut timing)) + .exchange_inner_mode( + signed, + membership, + None, + &scope, + false, + Some(&mut timing), + landing, + ) .await?; - let transport::RelayResponse::Complete(response) = response else { - return Err(Error::Internal); + let response = match response { + transport::RelayResponse::Complete(response) => response, + // Verified by exchange_inner_mode against `binding`. + transport::RelayResponse::Landed(landed) => { + scope.check()?; + return Ok(Delivered::Landed(landed)); + } + transport::RelayResponse::Http { .. } => return Err(Error::Internal), }; scope.check()?; let response = observer.result( @@ -741,7 +868,7 @@ impl Requester { .await, )?; timing.success(&response); - Ok(response) + Ok(Delivered::Response(response)) }) } } @@ -782,8 +909,9 @@ impl Requester { relay: Option>>, scope: &'a RequestScope, ) -> Operation<'a, transport::RelayResponse> { - self.exchange_inner_mode(request, membership, relay, scope, false, None) + self.exchange_inner_mode(request, membership, relay, scope, false, None, None) } + #[allow(clippy::too_many_arguments)] // Keep transport, routing, and landing inputs explicit. fn exchange_inner_mode<'a>( &'a self, request: SignedRequest, @@ -792,6 +920,7 @@ impl Requester { scope: &'a RequestScope, direct_http: bool, timing: Option<&'a mut PageTiming<'_>>, + landing: Option, ) -> Operation<'a, transport::RelayResponse> { let (network, paths, health, forwarding, transfers) = match self { Self::Network { @@ -877,6 +1006,7 @@ impl Requester { receive_permit, socket_failure.clone(), timing, + landing, &scope, ) .await; @@ -888,7 +1018,9 @@ impl Requester { }; use crate::topology::LinkOutcome; let outcome = match &response { - Ok(transport::RelayResponse::Complete(_)) => Some(LinkOutcome::Success), + Ok(transport::RelayResponse::Complete(_) | transport::RelayResponse::Landed(_)) => { + Some(LinkOutcome::Success) + } Err(_) if permit.is_none() && socket_failure.get() => Some(LinkOutcome::Refused), // Unavailable/deadline/protocol errors can arise locally or at a // downstream node. Do not blame an immediate peer without evidence. @@ -906,7 +1038,9 @@ impl Requester { { Outcome::Neutral } - Ok(transport::RelayResponse::Complete(_)) => Outcome::Verified, + Ok( + transport::RelayResponse::Complete(_) | transport::RelayResponse::Landed(_), + ) => Outcome::Verified, Err(Error::Overloaded) => Outcome::LocalPressure, _ => Outcome::Neutral, }); @@ -934,6 +1068,15 @@ impl Requester { .verify_response_async(response, binding, scope) .await .map(|verified| transport::RelayResponse::Complete(verified.into_signed())), + transport::RelayResponse::Landed(landed) => { + forwarding.verify_landed( + &landed.authentication, + &landed.metadata, + &landed.envelope, + binding, + )?; + Ok(transport::RelayResponse::Landed(landed)) + } transport::RelayResponse::Http { authentication, mut connection, diff --git a/cmd/racer-dataplane/src/peer/forwarding.rs b/cmd/racer-dataplane/src/peer/forwarding.rs index ce97e15af0..a205d3ed42 100644 --- a/cmd/racer-dataplane/src/peer/forwarding.rs +++ b/cmd/racer-dataplane/src/peer/forwarding.rs @@ -576,6 +576,28 @@ impl Forwarding { let (origin, forwarders) = self.verify_response_parts(&response, request)?; Ok(Self::finish_response(response, request, origin, forwarders)) } + /// Like [`Self::verify_response`] for a page that landed in GPU memory: + /// verify the signed head for `metadata` and `envelope`. The GPU checks + /// the ciphertext itself with the AEAD tag. + pub(crate) fn verify_landed( + &self, + authentication: &ForwardedHead, + metadata: &crate::model::ObjectMetadata, + envelope: &crate::model::PageEnvelope, + request: &RequestBinding, + ) -> Result<()> { + let path = protocol::decode_nodes( + field(&authentication.original.head, "racer-response-path")?.as_bytes(), + )?; + let expected = protocol::landed_response_head( + metadata, + envelope, + &signed_digest(&request.original)?, + &path, + )?; + self.verify_response_head(authentication, &expected, request) + .map(|_| ()) + } fn verify_response_parts( &self, response: &SignedResponse, diff --git a/cmd/racer-dataplane/src/peer/protocol.rs b/cmd/racer-dataplane/src/peer/protocol.rs index e11a322ec6..2145d4b2a3 100644 --- a/cmd/racer-dataplane/src/peer/protocol.rs +++ b/cmd/racer-dataplane/src/peer/protocol.rs @@ -768,6 +768,22 @@ pub fn response_head( Ok(head) } +/// The response head for a page whose ciphertext a NIC wrote straight into +/// GPU landing memory. It matches [`response_head`] for +/// [`PeerResponse::Page`]; only the metadata and envelope are signed, never +/// the ciphertext bytes, so no host copy is needed to check it. +pub(crate) fn landed_response_head( + metadata: &ObjectMetadata, + envelope: &crate::model::PageEnvelope, + request_digest: &[u8; 32], + path: &[NodeId], +) -> Result { + let mut head = response_prefix(request_digest, path)?; + page_fields(&mut head, metadata, envelope)?; + response_tail(&mut head, "page", u64::from(envelope.ciphertext_length)); + Ok(head) +} + /// Start a canonical response with its retained request binding and reverse path. fn response_prefix(binding: &[u8; 32], path: &[NodeId]) -> Result { let mut head = MessageHead { diff --git a/cmd/racer-dataplane/src/peer/server.rs b/cmd/racer-dataplane/src/peer/server.rs index a3b287c875..0eb85eda64 100644 --- a/cmd/racer-dataplane/src/peer/server.rs +++ b/cmd/racer-dataplane/src/peer/server.rs @@ -519,6 +519,10 @@ impl PeerServer { return result; } Ok((super::transport::RelayResponse::Complete(response), _)) => response, + // A relay never asks for landing. + Ok((super::transport::RelayResponse::Landed(_), _)) => { + return Err(Error::Internal); + } Err(Error::Overloaded) => self .forwarding .sign_response(&binding, PeerResponse::Overloaded)?, diff --git a/cmd/racer-dataplane/src/peer/tests.rs b/cmd/racer-dataplane/src/peer/tests.rs index 141a5e4cf0..dd8d376179 100644 --- a/cmd/racer-dataplane/src/peer/tests.rs +++ b/cmd/racer-dataplane/src/peer/tests.rs @@ -351,6 +351,7 @@ mod body_progress { None, Rc::new(Cell::new(false)), Some(&mut timing), + None, &scope, ); let result = loop { @@ -1309,6 +1310,7 @@ mod encrypted_http { None, Rc::new(Cell::new(false)), None, + None, turn, ) .await; @@ -1708,6 +1710,7 @@ mod requester_safety { matches!(response.response, PeerResponse::Overloaded), case == "overloaded" ), + RelayResponse::Landed(_) => panic!("nothing asked for landing"), } } assert_eq!(metrics.count(Event::PeerProbe), u64::from(case != "direct")); diff --git a/cmd/racer-dataplane/src/peer/tests/protocol_socket.rs b/cmd/racer-dataplane/src/peer/tests/protocol_socket.rs index 7b5f4f2b18..c4affa2460 100644 --- a/cmd/racer-dataplane/src/peer/tests/protocol_socket.rs +++ b/cmd/racer-dataplane/src/peer/tests/protocol_socket.rs @@ -1396,6 +1396,7 @@ fn real_http_ciphertext_fragmentation_pool_reuse_and_truncation() { None, Rc::new(std::cell::Cell::new(false)), None, + None, &scope, ) .await; diff --git a/cmd/racer-dataplane/src/peer/transport.rs b/cmd/racer-dataplane/src/peer/transport.rs index 105a2ec782..a23af0ce40 100644 --- a/cmd/racer-dataplane/src/peer/transport.rs +++ b/cmd/racer-dataplane/src/peer/transport.rs @@ -403,13 +403,27 @@ impl Transfers { .await?; Ok((written.lease, true)) } + /// Whether a rail of this node has a landing buffer on GPU `ordinal`. + pub(crate) fn landing_ready(&self, ordinal: i32) -> bool { + self.links.as_ref().is_some_and(|links| { + links + .active() + .iter() + .any(|rail| links.landing_ordinal(rail.rail) == Some(ordinal)) + }) + } /// Receiver: when the hop plan is RDMA and the link is ready, grant a page /// window in-band and sign it into the request's `Accept` control. Any /// recoverable failure means a plain HTTP request. + /// + /// With `landing`, the GPU of a `gpu-read` that asked for this fixed page, + /// the grant is over a slot of that GPU's landing buffer when this rail + /// has one and a slot is free. Otherwise it is a host grant as usual. pub(super) async fn accept_native( &self, request: &SignedRequest, plan: TransportPlan, + landing: Option, scope: &RequestScope, ) -> Result> { let TransportPlan::Rdma { rail } = plan else { @@ -436,10 +450,34 @@ impl Transfers { scope, rail, )?; - let pending = match links - .grant(&peer, rail, binding.transfer, &native_scope(scope)) - .await - { + let landed = match landing { + Some(ordinal) + if matches!( + request.request.operation, + crate::peer::protocol::Operation::Page { .. } + ) && links.landing_ordinal(rail) == Some(ordinal) => + { + match links + .grant_landing(&peer, rail, binding.transfer, ordinal, &native_scope(scope)) + .await + { + Ok(pending) => Some(pending), + // No free landing slot: grant host memory instead. + Err(error) if native_failure(error, scope) => None, + Err(error) => return Err(error), + } + } + _ => None, + }; + let granted = match landed { + Some(pending) => Ok(pending), + None => { + links + .grant(&peer, rail, binding.transfer, &native_scope(scope)) + .await + } + }; + let pending = match granted { Ok(pending) => pending, Err(error) if native_failure(error, scope) => { links.record_http_fallback(); @@ -522,9 +560,12 @@ impl Transfers { let read = self.io.read_body_exact(connection, buffer, scope).await?; Ok((read.lease, read.buffer)) } - /// Receiver: read the sender's one signed outcome. `Complete` means the page - /// is in the granted window; anything after that is `Io`, because the HTTP - /// body was never sent. `Finish` carries the HTTP body. + /// Receiver: read the sender's one signed outcome. `Complete` means the + /// page is in the granted window; anything after that is `Io`, because + /// the HTTP body was never sent. `Finish` carries the HTTP body. A landing + /// grant ([`Self::accept_native`] with a GPU) whose page the sender wrote + /// comes back as [`RelayResponse::Landed`]; any other page as + /// [`RelayResponse::Complete`]. #[allow(clippy::too_many_arguments)] // Preserve explicit authenticated transfer inputs. pub(super) async fn receive_native( &self, @@ -538,7 +579,7 @@ impl Transfers { membership: &std::sync::Arc, length: usize, scope: &RequestScope, - ) -> Result { + ) -> Result { binding.response = envelope_digest(&authentication)?; let (verified, phase) = binding.verify( &self.signatures, @@ -572,7 +613,49 @@ impl Transfers { drop(pending); return self .receive_fallback(connection, authentication, length, scope) - .await; + .await + .map(RelayResponse::Complete); + } + if pending.is_landing() { + let landed = pending + .finish_landed(&verified, &envelope, &native_scope(scope)) + .await?; + #[cfg(test)] + self.native_completions + .set(self.native_completions.get() + 1); + // Only a fixed page request asks for landing. The signed head must + // describe exactly this page; the requester verifies the chain + // against its binding, and the GPU checks the tag. + let original = &authentication.original.head; + if crate::peer::protocol::field(original, "racer-outcome")? != "page" { + return Err(Error::Unauthorized); + } + let request_digest = crate::peer::protocol::decode_binary( + crate::peer::protocol::field(original, "racer-request-binding")?.as_bytes(), + )? + .try_into() + .map_err(|_| Error::InvalidRequest)?; + let path = crate::peer::protocol::decode_nodes( + crate::peer::protocol::field(original, "racer-response-path")?.as_bytes(), + )?; + crate::peer::protocol::agrees( + original, + &crate::peer::protocol::landed_response_head( + &metadata, + &envelope, + &request_digest, + &path, + )?, + false, + )?; + let mut connection = connection; + connection.finish_exchange()?; + return Ok(RelayResponse::Landed(Box::new(LandedResponse { + authentication, + metadata, + envelope, + landed, + }))); } let (admission, _) = &self.wire; // The arrival stays in registered memory; retaining paths copy it out. @@ -615,10 +698,10 @@ impl Transfers { )?; let mut connection = connection; connection.finish_exchange()?; - Ok(SignedResponse { + Ok(RelayResponse::Complete(SignedResponse { authentication, response, - }) + })) } async fn receive_fallback( &self, @@ -964,12 +1047,28 @@ fn observe_read( result } +/// A peer page whose ciphertext a NIC wrote into a GPU landing buffer, with +/// its signed response envelope. The requester verifies the signatures +/// against its outstanding binding, as for any response, and the GPU checks +/// the tag. There are no host bytes: this never reaches memory, disk, or +/// another reader. +pub struct LandedResponse { + pub authentication: crate::peer::forwarding::ForwardedHead, + pub metadata: crate::model::ObjectMetadata, + pub envelope: crate::model::PageEnvelope, + pub landed: crate::gpu::LandedCiphertext, +} + /// Internal transport result: native delivery stays materialized; HTTP relay /// delivery owns an unfinished connection and its exact opaque body framing. #[allow(clippy::large_enum_variant)] // Keep completed delivery inline without another allocation. pub enum RelayResponse { Complete(SignedResponse), + /// A page in a GPU landing buffer. Only a requester that asked for + /// landing gets this, and only that requester's sealed read uses it. + Landed(Box), + Http { authentication: crate::peer::forwarding::ForwardedHead, @@ -1074,6 +1173,7 @@ impl Transfers { receive_permit: Option>, failure: Rc>, mut timing: Option<&'a mut super::PageTiming<'_>>, + landing: Option, scope: &'a RequestScope, ) -> Operation<'a, RelayResponse> { Box::pin(async move { @@ -1159,7 +1259,7 @@ impl Transfers { // Native response verification requires an authenticated membership lease. let granting = now(); let native = if membership.is_some() { - self.accept_native(&request, plan, scope).await? + self.accept_native(&request, plan, landing, scope).await? } else { None }; @@ -1230,8 +1330,7 @@ impl Transfers { length, scope, ) - .await - .map(RelayResponse::Complete); + .await; if received.is_ok() { self.record_rdma_stage(Event::RdmaArrivalCount, Event::RdmaArrivalNs, arriving); } @@ -1834,6 +1933,8 @@ mod tests { receiver_site: Option<&'static str>, /// The sender's link breaks before it writes. sender_down: bool, + /// The receiver grants a slot of a simulated GPU's landing buffer. + landing: bool, } struct PageOutcome { @@ -1841,6 +1942,8 @@ mod tests { result: Result>, /// The received ciphertext was still in registered memory. registered: bool, + /// The page landed in the GPU landing buffer. + landed: bool, /// Free buffers while the response was still held. held: (Option, Option), sent: Result, @@ -2007,17 +2110,27 @@ mod tests { }; let sender_membership = page_membership(&signers, case.sender_site); let receiver_membership = page_membership(&signers, case.receiver_site); + let gpu = case.landing.then(|| { + let helper = crate::gpu::testing::helper(&[None]); + let bytes = (crate::rdma::BUFFER_LEN as u64) + .next_multiple_of(crate::config::GPU_STAGING_UNIT); + crate::rdma::links::tests::attach_landing(&mut w, &helper, bytes); + assert_eq!(w.a.links.landing_ordinal(RailId(7)), Some(0)); + helper + }); // The receiver grants in-band and names the grant in its signed Accept. - let pending = crate::rdma::links::tests::drive( - &mut w.a.owner, - &mut w.b.owner, - receiver.links.as_ref().unwrap().grant( - signers[2].node(), - RailId(7), - binding.transfer, - &scope, - ), - ) + let links = receiver.links.as_ref().unwrap(); + let pending = crate::rdma::links::tests::drive(&mut w.a.owner, &mut w.b.owner, async { + if case.landing { + links + .grant_landing(signers[2].node(), RailId(7), binding.transfer, 0, &scope) + .await + } else { + links + .grant(signers[2].node(), RailId(7), binding.transfer, &scope) + .await + } + }) .unwrap(); let accept = binding .sign( @@ -2073,7 +2186,8 @@ mod tests { return receiver .wire .1 - .response(auth, body.bytes()?.to_vec(), &scope); + .response(auth, body.bytes()?.to_vec(), &scope) + .map(RelayResponse::Complete); }; receiver .receive_native( @@ -2157,17 +2271,40 @@ mod tests { let held = free(&w); let registered = matches!( &result, - Ok(SignedResponse { response: PeerResponse::Page { ciphertext, .. }, .. }) - if ciphertext.is_registered() + Ok(RelayResponse::Complete(SignedResponse { + response: PeerResponse::Page { ciphertext, .. }, + .. + })) if ciphertext.is_registered() ); - let result = result.map(|response| match &response.response { - PeerResponse::Page { ciphertext, .. } => ciphertext.bytes().to_vec(), + let landed = matches!(&result, Ok(RelayResponse::Landed(_))); + let result = result.map(|response| match response { + RelayResponse::Complete(SignedResponse { + response: PeerResponse::Page { ciphertext, .. }, + .. + }) => ciphertext.bytes().to_vec(), + RelayResponse::Landed(landed) => { + // Read what the NIC wrote, through the landing dma-buf. + use std::os::unix::fs::FileExt; + assert_eq!(landed.envelope.ciphertext_length, 144); + let buffer = w.a.links.node().landing(RailId(7)).unwrap(); + let file = std::fs::File::from( + rdma_verbs::DmabufMemory::fd(&*buffer) + .try_clone_to_owned() + .unwrap(), + ); + let mut bytes = vec![0; landed.landed.len() as usize]; + file.read_exact_at(&mut bytes, landed.landed.offset()) + .unwrap(); + bytes + } _ => panic!("expected page"), }); + drop(gpu); crate::rdma::links::tests::pump(&mut w.a.owner, &mut w.b.owner); PageOutcome { result, registered, + landed, held, sent, completed: sender.native_completed.get(), @@ -2208,6 +2345,48 @@ mod tests { } } + #[test] + /// A landing grant over a ready link: the sender writes into the + /// receiver's GPU landing buffer, the receiver checks the signed head + /// for exactly this page and returns it as landed, and no host buffer + /// is used on the receiver. + fn simulated_landing_grant_lands_the_page_in_gpu_memory() { + for reverse in [false, true] { + let outcome = page_roundtrip(PageCase { + reverse, + landing: true, + ..PageCase::default() + }); + assert_eq!(page_bytes(&outcome.result), vec![0x5a; 144]); + assert!(outcome.landed && !outcome.registered); + assert_eq!(outcome.sent, Ok(true)); + assert_eq!( + (outcome.completed, outcome.fallbacks, outcome.completions), + (1, 0, 1) + ); + // No host receive buffer was used. + assert_eq!(outcome.held, (Some(2), Some(2))); + assert_eq!(outcome.free, (Some(2), Some(2))); + } + } + + #[test] + /// The sender may still answer a landing grant over HTTP (here its + /// link is down): the page arrives in host memory as usual. + fn simulated_landing_grant_falls_back_to_http() { + let outcome = page_roundtrip(PageCase { + landing: true, + sender_down: true, + ..PageCase::default() + }); + assert_eq!(page_bytes(&outcome.result), vec![0x5a; 144]); + assert!(!outcome.landed); + assert_eq!( + (outcome.completed, outcome.fallbacks, outcome.completions), + (0, 1, 0) + ); + } + #[test] fn simulated_sender_rejects_cross_site_and_releases_the_grant() { for site in ["site2", ""] { diff --git a/cmd/racer-dataplane/src/rdma/links.rs b/cmd/racer-dataplane/src/rdma/links.rs index 75484386d4..4ffa35544f 100644 --- a/cmd/racer-dataplane/src/rdma/links.rs +++ b/cmd/racer-dataplane/src/rdma/links.rs @@ -93,6 +93,41 @@ pub struct LinkNode { /// Source of reservation tokens for `initiating`. reservations: std::sync::atomic::AtomicU64, + + /// GPU helper and per-GPU landing bytes, when `gpu-read` pages may land + /// in GPU memory (GPUDirect RDMA). + gpu_landing: Mutex, u64)>>, + + /// The GPU landing buffer registered on each rail this round. + landings: Mutex>>, +} + +/// Pick at most one GPU per rail for landing, and each GPU for at most one +/// rail. A rail first takes a free GPU on its NUMA node, then any free GPU. +/// `rails` and `gpus` give each one's NUMA node, if known. PCI distance is +/// not used: only NUMA locality is known on both sides. +pub fn assign_landing( + rails: &[(RailId, Option)], + gpus: &[(i32, Option)], +) -> Vec<(RailId, i32)> { + let mut free: Vec<(i32, Option)> = gpus.to_vec(); + let mut assigned = Vec::new(); + let mut later = Vec::new(); + for &(rail, numa) in rails { + match free + .iter() + .position(|&(_, node)| numa.is_some() && node == numa) + { + Some(index) => assigned.push((rail, free.remove(index).0)), + None => later.push(rail), + } + } + for rail in later { + if !free.is_empty() { + assigned.push((rail, free.remove(0).0)); + } + } + assigned } /// One setup this node initiated: reserved before the owner answers, then @@ -150,7 +185,20 @@ impl LinkNode { .store(true, std::sync::atomic::Ordering::Release); } + /// Let `gpu-read` pages land in GPU memory: on each activation, give + /// each rail one GPU's landing buffer of `bytes` (see + /// [`assign_landing`]). 0 bytes turns landing off. + pub fn set_gpu_landing(&self, helper: Arc, bytes: u64) { + *Self::lock(&self.gpu_landing) = (bytes != 0).then_some((helper, bytes)); + } + + /// The GPU landing buffer on `rail` this round, if any. + pub fn landing(&self, rail: RailId) -> Option> { + Self::lock(&self.landings).get(&rail.0).cloned() + } + fn reset(&self) { + Self::lock(&self.landings).clear(); Self::lock(&self.active).clear(); self.settled .store(false, std::sync::atomic::Ordering::Release); @@ -343,6 +391,9 @@ impl Links { let result = self .activate_inner(publication, admission, registered_bytes, scope) .await; + if let Ok(rails) = &result { + self.attach_landings(rails, scope).await; + } match &result { Ok(rails) => self.node.settle(rails.clone()), Err(Error::Cancelled | Error::DeadlineExceeded) => {} @@ -431,6 +482,62 @@ impl Links { .collect()) } + /// Register GPU landing buffers on the activated `rails`. Failures only + /// log: pages then land in host memory as usual. + async fn attach_landings(&self, rails: &[RailMapping], scope: &RequestScope) { + let Some((helper, bytes)) = LinkNode::lock(&self.node.gpu_landing).clone() else { + return; + }; + if bytes < BUFFER_LEN as u64 { + eprintln!( + "racer-dataplane: stage=gpu-landing state=disabled reason=too-small bytes={bytes}" + ); + return; + } + let gpus: Vec<(i32, Option)> = helper + .devices() + .iter() + .filter(|device| helper.decrypts(device.ordinal)) + .map(|device| (device.ordinal, device.numa_node.map(|n| n as usize))) + .collect(); + let rails: Vec<(RailId, Option)> = + rails.iter().map(|m| (m.rail, m.numa_node)).collect(); + for (rail, ordinal) in assign_landing(&rails, &gpus) { + let landing = match helper.landing(ordinal, bytes, scope).await { + Ok(landing) => landing, + Err(error) => { + eprintln!( + "racer-dataplane: stage=gpu-landing rail={} ordinal={ordinal} state=failed reason={error}", + rail.0 + ); + continue; + } + }; + let memory = landing.clone() as rdma_verbs::DeviceMemory; + match self + .wait(self.client.attach_device(rail.0, memory), scope) + .await + { + Ok(slots) => { + eprintln!( + "racer-dataplane: stage=gpu-landing rail={} ordinal={ordinal} state=enabled slots={slots}", + rail.0 + ); + LinkNode::lock(&self.node.landings).insert(rail.0, landing); + } + Err(error) => eprintln!( + "racer-dataplane: stage=gpu-landing rail={} ordinal={ordinal} state=failed reason={error}", + rail.0 + ), + } + } + } + + /// The GPU whose landing buffer is on `rail`, if any. + pub fn landing_ordinal(&self, rail: RailId) -> Option { + self.node.landing(rail).map(|landing| landing.ordinal()) + } + /// Close the node: every link breaks and the owner drains. A later /// [`Self::activate`] reopens once drained. pub fn close(&self) { @@ -682,6 +789,52 @@ impl Links { rail, transfer, arrival: granted.arrival, + landing: None, + }) + } + + /// Receiver: like [`Self::grant`], but the window is over a slot of the + /// GPU landing buffer on `rail`, which must be on GPU `ordinal`. Fails + /// with `Unavailable` if it is not, and `Overloaded` if no slot is free; + /// the caller then grants host memory. + pub async fn grant_landing( + &self, + peer: &NodeId, + rail: RailId, + transfer: TransferId, + ordinal: i32, + scope: &RequestScope, + ) -> Result { + scope.check()?; + let landing = self + .node + .landing(rail) + .filter(|landing| landing.ordinal() == ordinal) + .ok_or(Error::Unavailable)?; + let (epoch, link) = self.ready(peer, rail).ok_or(Error::Unavailable)?; + let length = MAX_CIPHERTEXT as u32; + let granted = self + .wait( + self.client.grant_device( + link_key(peer, rail), + epoch, + transfer.0, + length, + scope.deadline.0, + ), + scope, + ) + .await?; + Ok(PendingReceive { + metrics: self.metrics.clone(), + retry: self.retry.clone(), + link, + tag: granted.tag, + length, + rail, + transfer, + arrival: granted.arrival, + landing: Some(landing), }) } @@ -791,9 +944,65 @@ pub struct PendingReceive { rail: RailId, transfer: TransferId, arrival: Op, + + /// The GPU landing buffer the grant is over, for a landing grant. + landing: Option>, } impl PendingReceive { + /// True for a grant over a GPU landing slot ([`Links::grant_landing`]). + pub fn is_landing(&self) -> bool { + self.landing.is_some() + } + + /// Check the signed completion against this grant and the envelope. + fn check_completion(&self, head: &VerifiedHead, envelope: &PageEnvelope) -> Result { + validate_envelope(envelope)?; + let length = envelope.ciphertext_length; + if signed_value(head, COMPLETION_HEADER, COMPLETE_LEN)? + != completion_value(self.link, self.tag, length, self.transfer) + || length > self.length + { + return Err(Error::Unauthorized); + } + Ok(length) + } + + /// Like [`Self::finish`] for a landing grant: the ciphertext stays in GPU + /// memory and comes back as [`crate::gpu::LandedCiphertext`]. The CPU + /// never reads it; the GPU checks the tag before writing plaintext. + pub async fn finish_landed( + self, + head: &VerifiedHead, + envelope: &PageEnvelope, + scope: &RequestScope, + ) -> Result { + scope.check()?; + let length = self.check_completion(head, envelope)?; + let landing = self.landing.clone().ok_or(Error::Internal)?; + let arrival = self + .retry + .wait(scope, poll_fn(|cx| self.arrival.poll(cx))) + .await; + let page = match arrival { + Ok(Arrival::Device(page)) => page, + Ok(Arrival::Released | Arrival::Page(_)) => return Err(Error::Io), + Err(error @ Error::Cancelled) => return Err(error), + Err(_) => { + scope.check()?; + return Err(Error::Io); + } + }; + if page.len() != length as usize { + return Err(Error::InvalidRange); + } + if let Some(metrics) = &self.metrics { + metrics.record(crate::telemetry::Event::RdmaPagesReceived, 1); + metrics.record(crate::telemetry::Event::RdmaReceivedBytes, length.into()); + metrics.record(crate::telemetry::Event::RdmaPagesLanded, 1); + } + crate::gpu::LandedCiphertext::new(&landing, page) + } /// Base64 value of the signed `GRANT_HEADER` extension. pub fn header_value(&self) -> Vec { STANDARD @@ -834,14 +1043,7 @@ impl PendingReceive { scope: &RequestScope, ) -> Result { scope.check()?; - validate_envelope(&envelope)?; - let length = envelope.ciphertext_length; - if signed_value(head, COMPLETION_HEADER, COMPLETE_LEN)? - != completion_value(self.link, self.tag, length, self.transfer) - || length > self.length - { - return Err(Error::Unauthorized); - } + let length = self.check_completion(head, &envelope)?; let reservation = admission.reserve( Some(&envelope.page.version.object.cache), ResourceClass::Ciphertext, @@ -853,7 +1055,8 @@ impl PendingReceive { .await; let page = match arrival { Ok(Arrival::Page(page)) => page, - Ok(Arrival::Released) => return Err(Error::Io), + // A landing grant finishes with `finish_landed`. + Ok(Arrival::Released | Arrival::Device(_)) => return Err(Error::Io), Err(error @ Error::Cancelled) => return Err(error), Err(_) => { scope.check()?; diff --git a/cmd/racer-dataplane/src/rdma/links/tests.rs b/cmd/racer-dataplane/src/rdma/links/tests.rs index 930a141e80..cc6a1ccf8c 100644 --- a/cmd/racer-dataplane/src/rdma/links/tests.rs +++ b/cmd/racer-dataplane/src/rdma/links/tests.rs @@ -166,7 +166,7 @@ pub(crate) fn receive_registered( drive!(w, w.b.links.send(&a, &grant, page, &scope)).unwrap(); match drive!(w, poll_fn(|cx| pending.arrival.poll(cx))).unwrap() { Arrival::Page(page) => page, - Arrival::Released => panic!("expected a page"), + Arrival::Released | Arrival::Device(_) => panic!("expected a page"), } } @@ -382,7 +382,7 @@ fn setup_page_both_directions_and_release() { let arrival = drive!(w, poll_fn(|cx| pending.arrival.poll(cx))).unwrap(); match arrival { Arrival::Page(page) => assert_eq!(page.bytes(), &[0x5a; 35]), - Arrival::Released => panic!("expected a page"), + Arrival::Released | Arrival::Device(_) => panic!("expected a page"), } // Release path: the receiver sees `Released` and the buffer is free. let pending = drive!(w, receiver.grant(from, RAIL, TransferId([4; 16]), &scope)).unwrap(); @@ -634,7 +634,7 @@ mod send_failures { fn arrived(w: &mut World, pending: &PendingReceive) -> Vec { match drive!(w, poll_fn(|cx| pending.arrival.poll(cx))).unwrap() { Arrival::Page(page) => page.bytes().to_vec(), - Arrival::Released => panic!("expected a page"), + Arrival::Released | Arrival::Device(_) => panic!("expected a page"), } } @@ -796,7 +796,7 @@ mod send_in_place { let arrival = drive!(w, poll_fn(|cx| pending.arrival.poll(cx))).unwrap(); match arrival { Arrival::Page(page) => assert_eq!(page.bytes(), &[0xc3; 35][..]), - Arrival::Released => panic!("expected a page"), + Arrival::Released | Arrival::Device(_) => panic!("expected a page"), } pump!(w); assert!( @@ -847,3 +847,151 @@ mod send_in_place { assert_eq!(ma.count(Event::RdmaHttpFallback), 1); } } + +#[test] +/// Rails take a free GPU on their NUMA node first, then any free GPU; each +/// GPU serves one rail at most. +fn assign_landing_prefers_numa_then_any() { + let rails = [ + (RailId(1), Some(1)), + (RailId(2), Some(0)), + (RailId(3), None), + (RailId(4), Some(0)), + ]; + let gpus = [(0, Some(0)), (1, Some(1)), (2, None)]; + assert_eq!( + assign_landing(&rails, &gpus), + [(RailId(1), 1), (RailId(2), 0), (RailId(3), 2)] + ); + assert_eq!(assign_landing(&rails, &[]), []); + assert_eq!( + assign_landing(&[(RailId(9), Some(3))], &[(5, Some(1))]), + [(RailId(9), 5)] + ); +} + +/// Register a simulated GPU's landing buffer on `a`'s rail. +pub(crate) fn attach_landing(w: &mut World, helper: &Arc, bytes: u64) { + w.a.links.node().set_gpu_landing(helper.clone(), bytes); + let rails = w.a.links.active(); + let scope = scope(); + // The GPU helper answers from its own thread: poll with short sleeps. + let mut future = std::pin::pin!(w.a.links.attach_landings(&rails, &scope)); + let mut cx = Context::from_waker(futures::task::noop_waker_ref()); + for _ in 0..5000 { + if future.as_mut().poll(&mut cx).is_ready() { + return; + } + pump(&mut w.a.owner, &mut w.b.owner); + std::thread::sleep(Duration::from_millis(1)); + } + panic!("landing attach did not finish"); +} + +#[test] +/// A landing grant takes a slot of the GPU landing buffer: the peer's write +/// lands in GPU memory, the CPU path never sees it, and the slot is held +/// until the landed page drops. A landing grant for another GPU, or with no +/// free slot, is refused so the caller grants host memory instead. +fn landing_grant_receives_into_gpu_memory() { + use std::os::unix::fs::FileExt; + let helper = crate::gpu::testing::helper(&[None]); + let mut w = world(); + let link = link_up(&mut w); + // One slot only. + let bytes = (BUFFER_LEN as u64).next_multiple_of(crate::config::GPU_STAGING_UNIT); + attach_landing(&mut w, &helper, bytes); + assert_eq!(w.a.links.landing_ordinal(RAIL), Some(0)); + assert_eq!(w.b.links.landing_ordinal(RAIL), None); + let (a, b) = (node_id(1), node_id(2)); + let scope = scope(); + assert_eq!( + drive!( + w, + w.a.links + .grant_landing(&b, RAIL, TransferId([3; 16]), 1, &scope) + ) + .err(), + Some(Error::Unavailable), + "landing buffer is on another GPU" + ); + let pending = drive!( + w, + w.a.links + .grant_landing(&b, RAIL, TransferId([3; 16]), 0, &scope) + ) + .unwrap(); + assert!(pending.is_landing()); + assert_eq!(pending.link(), link); + assert!( + drive!( + w, + w.a.links + .grant_landing(&b, RAIL, TransferId([4; 16]), 0, &scope) + ) + .is_err(), + "the only slot is taken" + ); + let header = STANDARD.decode(pending.header_value()).unwrap(); + let grant = AuthenticatedGrant::decode(&header, RAIL, TransferId([3; 16])).unwrap(); + let page = page(&w.b.admission.clone(), 35, 0x6b); + drive!(w, w.b.links.send(&a, &grant, &page, &scope)).unwrap(); + let device = match drive!(w, poll_fn(|cx| pending.arrival.poll(cx))).unwrap() { + Arrival::Device(page) => page, + _ => panic!("expected a device page"), + }; + // The bytes are in the GPU buffer, read here through its dma-buf. + let landing = w.a.links.node().landing(RAIL).unwrap(); + let mut bytes = [0; 35]; + let file = std::fs::File::from( + rdma_verbs::DmabufMemory::fd(&*landing) + .try_clone_to_owned() + .unwrap(), + ); + file.read_exact_at(&mut bytes, device.offset()).unwrap(); + assert_eq!(bytes, [0x6b; 35]); + let landed = crate::gpu::LandedCiphertext::new(&landing, device).unwrap(); + assert_eq!((landed.ordinal(), landed.len()), (0, 35)); + // The host pool was not used. + assert_eq!(w.a.links.buffers(RAIL, BufferState::Free), Some(2)); + drop(landed); + pump!(w); + let again = drive!( + w, + w.a.links + .grant_landing(&b, RAIL, TransferId([5; 16]), 0, &scope) + ) + .unwrap(); + let header = STANDARD.decode(again.header_value()).unwrap(); + let grant = AuthenticatedGrant::decode(&header, RAIL, TransferId([5; 16])).unwrap(); + w.b.links.release(&a, &grant); + assert!(matches!( + drive!(w, poll_fn(|cx| again.arrival.poll(cx))), + Ok(Arrival::Released) + )); + // Closing the node forgets the landing buffer for the next round. + w.a.links.close(); + assert_eq!(w.a.links.landing_ordinal(RAIL), None); +} + +#[test] +/// No usable landing: too little memory, no GPU, or a failed registration +/// leaves the rail host-only. +fn landing_failures_leave_rails_host_only() { + let helper = crate::gpu::testing::helper(&[None]); + let mut w = world(); + attach_landing(&mut w, &helper, (BUFFER_LEN as u64) / 2); + assert_eq!(w.a.links.landing_ordinal(RAIL), None); + attach_landing(&mut w, &helper, 0); + assert_eq!(w.a.links.landing_ordinal(RAIL), None); + w._sim.fault( + rdma_verbs::simulation::Operation::RegisterDmabuf, + rdma_verbs::simulation::Fault::Reject, + ); + let bytes = (BUFFER_LEN as u64).next_multiple_of(crate::config::GPU_STAGING_UNIT); + attach_landing(&mut w, &helper, bytes); + assert_eq!(w.a.links.landing_ordinal(RAIL), None); + // Registration works on the next try, reusing the same GPU buffer. + attach_landing(&mut w, &helper, bytes); + assert_eq!(w.a.links.landing_ordinal(RAIL), Some(0)); +} diff --git a/cmd/racer-dataplane/src/read.rs b/cmd/racer-dataplane/src/read.rs index e1635d6479..1463011016 100644 --- a/cmd/racer-dataplane/src/read.rs +++ b/cmd/racer-dataplane/src/read.rs @@ -16,6 +16,7 @@ use crate::model::ByteRange; use crate::model::MetadataSelector; use crate::model::ObjectId; use crate::model::ObjectMetadata; +use crate::model::PAGE_BYTES; use crate::model::ResolvedRange; use crate::peer::forwarding::VerifiedRequest; use crate::peer::protocol::FetchMode; @@ -36,6 +37,7 @@ pub mod fill; pub mod flight; pub mod metadata; pub mod range_stream; +pub mod sealed; pub mod warm; /// Metadata and its optional stream, with no independent range to disagree with it. @@ -169,7 +171,9 @@ impl Coordinator { retain, } = request; let selector = match &kind { - ReadKind::Subscription { pin, .. } => pin + ReadKind::Subscription { pin, .. } + | ReadKind::HostRead { pin, .. } + | ReadKind::GpuRead { pin, .. } => pin .clone() .map_or(MetadataSelector::Fresh, MetadataSelector::Pinned), ReadKind::Head => MetadataSelector::Fresh, @@ -185,6 +189,21 @@ impl Coordinator { ) .await?; validate_metadata(&metadata, &origin.object, &selector)?; + // Direct reads have no client credits. The dataplane paces them + // with its own window, at least `range_window_pages`, and releases + // each page after writing it. + let direct_window = crate::client::DIRECT_WINDOW_PAGES.max(self.streams.window_pages()); + let kind = match kind { + ReadKind::HostRead { range, .. } | ReadKind::GpuRead { range, .. } => { + ReadKind::Subscription { + pin: None, + range, + page_credits: direct_window, + byte_credits: direct_window as u64 * PAGE_BYTES, + } + } + kind => kind, + }; match kind { ReadKind::Subscription { range, @@ -244,6 +263,8 @@ impl Coordinator { retain_until, }) } + // Mapped to Subscription above. + ReadKind::HostRead { .. } | ReadKind::GpuRead { .. } => Err(Error::Internal), } }) } diff --git a/cmd/racer-dataplane/src/read/candidates.rs b/cmd/racer-dataplane/src/read/candidates.rs index c5c1c898cf..8a9e696dc3 100644 --- a/cmd/racer-dataplane/src/read/candidates.rs +++ b/cmd/racer-dataplane/src/read/candidates.rs @@ -9,6 +9,7 @@ use crate::model::AttemptId; use crate::model::MetadataSelector; use crate::model::ObjectId; use crate::model::PageNumber; +use crate::peer::Delivered; use crate::peer::Requester; use crate::peer::forwarding::VerifiedResponse; use crate::peer::protocol::FetchMode; @@ -1426,6 +1427,82 @@ impl CandidatePolicy { request_mode: RequestMode, trail: Option<(&mut CandidateTrail, u8)>, ) -> Result { + match self + .request_delivered( + membership, + destination, + context, + operation, + mode, + scope, + budget, + remaining_opportunities, + request_mode, + trail, + None, + ) + .await? + { + Delivered::Response(response) => Ok(response), + Delivered::Landed(_) => Err(Error::Internal), + } + } + + /// One CopyOnly page request for a `gpu-read` on GPU `ordinal`: the page + /// may land in a GPU landing buffer (see [`Requester::request_landing`]). + /// The caller sends it to one provider only. + #[allow(clippy::too_many_arguments)] // Same route inputs as a plain request. + pub(crate) async fn request_landing( + &self, + membership: &std::sync::Arc, + destination: &NodeId, + context: &OriginContext, + page: &crate::model::PageId, + scope: &RequestScope, + budget: &mut AcquisitionBudget, + ordinal: i32, + ) -> Result { + let operation = PeerOperation::Page { + page: page.clone(), + mode: FetchMode::CopyOnly, + }; + self.request_delivered( + membership, + destination, + context, + &operation, + FetchMode::CopyOnly, + scope, + budget, + 2, + RequestMode::SharedRoute, + None, + Some(ordinal), + ) + .await + } + + /// Whether this node can land peer pages on GPU `ordinal`. + pub(crate) fn landing_ready(&self, ordinal: i32) -> bool { + self.peers.landing_ready(ordinal) + } + + /// Request a peer copy, optionally into a GPU landing buffer. + #[allow(clippy::too_many_arguments)] // Keep the optional borrowed trace separate from route state. + async fn request_delivered( + &self, + membership: &std::sync::Arc, + destination: &NodeId, + context: &OriginContext, + operation: &PeerOperation, + mode: FetchMode, + scope: &RequestScope, + budget: &mut AcquisitionBudget, + remaining_opportunities: u32, + request_mode: RequestMode, + trail: Option<(&mut CandidateTrail, u8)>, + landing: Option, + ) -> Result { let mut remote = [b'?'; 36]; if trail.is_some() && racer_crypto::identity::canonical_uuid(&destination.0) { remote.copy_from_slice(destination.0.as_bytes()); @@ -1547,12 +1624,18 @@ impl CandidatePolicy { }, }; let registration = scope.cancellation.subscribe()?; - let mut exchange = if matches!(request_mode, RequestMode::DirectHedge) { + let mut exchange: Operation<'_, Delivered> = if let Some(ordinal) = landing { self.peers - .request_direct(request, membership.clone(), &attempt_scope) + .request_landing(request, membership.clone(), ordinal, &attempt_scope) } else { - self.peers - .request(request, membership.clone(), &attempt_scope) + let exchange = if matches!(request_mode, RequestMode::DirectHedge) { + self.peers + .request_direct(request, membership.clone(), &attempt_scope) + } else { + self.peers + .request(request, membership.clone(), &attempt_scope) + }; + Box::pin(async move { exchange.await.map(Delivered::Response) }) }; let mut stopped = None; facts.site = CandidateSite::Exchange; @@ -1573,7 +1656,8 @@ impl CandidatePolicy { .await; facts.raw = Some(match &response { Err(error) => CandidateOutcome::Error(*error), - Ok(response) => match response.response() { + Ok(Delivered::Landed(_)) => CandidateOutcome::Success, + Ok(Delivered::Response(response)) => match response.response() { PeerResponse::Miss => CandidateOutcome::Miss, PeerResponse::Unavailable => { CandidateOutcome::ResponseError(Error::Unavailable) @@ -1605,7 +1689,8 @@ impl CandidatePolicy { links: budget.remaining_links(), }), ), - Ok(response) => { + Ok(Delivered::Landed(_)) => {} + Ok(Delivered::Response(response)) => { let error = match response.response() { PeerResponse::Overloaded => Some(Error::Overloaded), PeerResponse::Unavailable => Some(Error::Unavailable), diff --git a/cmd/racer-dataplane/src/read/dispatch.rs b/cmd/racer-dataplane/src/read/dispatch.rs index a7be805b9f..d6d25c5060 100644 --- a/cmd/racer-dataplane/src/read/dispatch.rs +++ b/cmd/racer-dataplane/src/read/dispatch.rs @@ -62,6 +62,37 @@ thread_local! { struct Mailbox { worker: WorkerId, inner: Arc, + /// Failed GPU verdicts the mailbox could not carry. The owning endpoint + /// applies them on its next poll so a corrupt copy is never left eligible. + failed_verdicts: std::sync::Mutex>, +} + +/// Bound on retained failed verdicts per worker. Each entry is one failed GPU +/// verification; beyond this the owner is far behind and the oldest are kept. +const FAILED_VERDICT_LIMIT: usize = 4096; + +impl Mailbox { + /// Retain a failed verdict for the owner. Returns false only when full. + fn retain_failed_verdict(&self, report: super::sealed::SealedReport) -> bool { + let mut pending = self + .failed_verdicts + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + if pending.len() >= FAILED_VERDICT_LIMIT { + return false; + } + pending.push(report); + true + } + + fn take_failed_verdicts(&self) -> Vec { + std::mem::take( + &mut *self + .failed_verdicts + .lock() + .unwrap_or_else(|poison| poison.into_inner()), + ) + } } impl std::ops::Deref for Mailbox { type Target = Handoff; @@ -117,6 +148,16 @@ enum Work { std::sync::Arc, PeerOriginContext, ), + /// One read window page for direct GPU delivery: like `Fetch`, but + /// ciphertext the GPU can verify is returned sealed. + FetchSealed( + PageId, + std::sync::Arc, + PeerOriginContext, + super::sealed::SealedTarget, + ), + /// The GPU's verdict on a sealed page this worker supplied. + SealedVerdict(super::sealed::SealedReport, bool), Publish(VersionMetadata), Retained(ObjectVersion), Peer(VerifiedRequest, std::sync::Arc), @@ -125,6 +166,7 @@ enum Value { Held(bool, std::time::SystemTime, HoldEpoch), Metadata(ObjectMetadata), Page(PageResult), + Fetched(super::sealed::Fetched), Cached(Option), Published, Retained(Option), @@ -365,6 +407,7 @@ impl WorkerDirectory { Arc::new(Mailbox { worker, inner: Arc::new(Handoff::new(capacity).expect("validated capacity")), + failed_verdicts: std::sync::Mutex::new(Vec::new()), }) }) .collect(), @@ -873,6 +916,88 @@ impl WorkerDirectory { Ok((value, budget)) })) } + /// Like [`Self::start_page`], but ciphertext the GPU behind `target` + /// can verify comes back sealed instead of decrypted on the CPU. + pub(crate) fn start_page_sealed( + &self, + page: PageId, + membership: std::sync::Arc, + context: &OriginContext, + scope: &RequestScope, + budget: AcquisitionBudget, + target: super::sealed::SealedTarget, + ) -> Result, AcquisitionBudget)>> { + if page.version.object != context.object { + return Err(Error::InvalidRequest); + } + let owner = self.page_owner(&page)?; + let receipt = self.submit( + owner, + Work::FetchSealed(page, membership, self.seal(context, scope)?, target), + scope, + Some(budget), + )?; + Ok(Box::pin(async move { + let completion = receipt.await?; + let budget = completion.budget.ok_or(Error::StaleFlight)?; + let value = completion.value.and_then(|value| match value { + Value::Fetched(value) => Ok(value), + _ => Err(Error::StaleFlight), + }); + Ok((value, budget)) + })) + } + + /// Tell a sealed page's owner what the GPU decided, in the background. + pub(crate) fn report_sealed( + &self, + report: super::sealed::SealedReport, + passed: bool, + scope: &RequestScope, + ) { + let Ok(owner) = self.page_owner(&report.page) else { + return; + }; + // A landed page has no host copy or disk mapping for its owner to act + // on, so its verdict is settled here and never takes a retained slot. + if self.is_local(owner) || !report.has_owner_state() { + if let Ok(local) = self.local() { + local.fill.sealed_verdict(report, passed); + } + return; + } + // A failed verdict must reach the owner, or the corrupt copy stays + // eligible. When the driver queue or mailbox cannot take it, retain + // it on the owner's mailbox for the owner's next poll. + let retain = |report: super::sealed::SealedReport| { + if !passed && let Ok(mailbox) = self.mailbox(owner) { + mailbox.retain_failed_verdict(report); + } + }; + let Ok(permit) = uring_runtime::drivers::reserve() else { + retain(report); + return; + }; + // The request may already be ending; the verdict must still arrive. + let deadline = uring_runtime::environment::now() + std::time::Duration::from_secs(5); + let Ok(detached) = RequestScope::new(scope.request, deadline) else { + retain(report); + return; + }; + let kept = (!passed).then(|| report.clone()); + match self.submit(owner, Work::SealedVerdict(report, passed), &detached, None) { + Ok(receipt) => permit.submit_detached(Box::pin(async move { + let _ = receipt.await; + Ok::<(), Error>(()) + })), + Err(_) => { + if let Some(report) = kept { + retain(report); + } + } + } + } + pub fn publish_metadata<'a>( &'a self, metadata: VersionMetadata, @@ -995,6 +1120,12 @@ impl WorkerEndpoint { self.directory.simulation_crash(); self.active.simulation_crash(); } + /// Apply failed GPU verdicts other workers could not send through the mailbox. + fn apply_failed_verdicts(&self) { + for report in self.mailbox.take_failed_verdicts() { + self.local.fill.sealed_verdict(report, false); + } + } pub fn stop_admission(&self) { self.directory.warm.stop(); self.mailbox.stop_admission(); @@ -1007,6 +1138,7 @@ impl WorkerEndpoint { pub fn drain<'a>(&'a mut self, scope: &'a RequestScope) -> Operation<'a, ()> { self.stop_admission(); Box::pin(std::future::poll_fn(move |cx| { + self.apply_failed_verdicts(); self.directory.warm.poll_releases( self.mailbox.worker, self.local.fill.dependencies.writer.index(), @@ -1027,6 +1159,7 @@ impl WorkerEndpoint { } /// Register a reactor waker and execute at most `work_budget` command/poll steps. pub fn poll(&mut self, cx: &mut Context<'_>, work_budget: usize) -> Result<()> { + self.apply_failed_verdicts(); self.directory.warm.poll_deadlines(); self.directory.warm.poll_releases( self.mailbox.worker, @@ -1229,6 +1362,30 @@ async fn execute( .await .map(Value::Page) } + Work::FetchSealed(page, membership, envelope, target) => { + if let Warmed::Local(result) = + local.streams.directory().warmed_page(&page, scope).await? + { + return Ok(Value::Fetched(super::sealed::Fetched::Page(*result))); + } + let context = local.open_context(envelope)?; + local + .fill + .acquire_sealed( + page, + membership, + &context, + scope, + budget.ok_or(Error::StaleFlight)?, + &target, + ) + .await + .map(Value::Fetched) + } + Work::SealedVerdict(report, passed) => { + local.fill.sealed_verdict(report, passed); + Ok(Value::Published) + } Work::Publish(metadata) => { local.metadata.publish_version(metadata)?; Ok(Value::Published) @@ -1461,6 +1618,147 @@ mod tests { } job.stop(); } + /// A page owned by `worker` in the two-worker test map. + fn page_owned_by(directory: &WorkerDirectory, worker: WorkerId) -> PageId { + (0..1024) + .map(|number| PageId { + version: version(), + number: crate::model::PageNumber(number), + }) + .find(|page| directory.page_owner(page).unwrap() == worker) + .unwrap() + } + fn failed_report(page: PageId) -> super::super::sealed::SealedReport { + super::super::sealed::SealedReport { + page: page.clone(), + copy: None, + token: Some(crate::store::ReadToken::for_test(page)), + } + } + /// A landed page's report: nothing for the owner to persist or drop. + fn landed_report(page: PageId) -> super::super::sealed::SealedReport { + super::super::sealed::SealedReport { + page, + copy: None, + token: None, + } + } + /// Deliver retained verdicts through the owner's endpoint and count them. + fn applied_by_owner(directory: &Arc) -> u64 { + let (owner, _fixture) = crate::read::tests::admission_coordinator(); + let before = owner + .fill + .metrics + .count(crate::telemetry::Event::CorruptMiss); + let mut endpoint = directory.install(WorkerId(1), owner.clone()).unwrap(); + endpoint.poll_budgeted(0).unwrap(); + let applied = owner + .fill + .metrics + .count(crate::telemetry::Event::CorruptMiss) + - before; + drop(endpoint); + applied + } + #[test] + fn failed_sealed_verdict_survives_full_driver_queue() { + let queue = Rc::new(uring_runtime::drivers::DriverQueue::new(1)); + let _queue = queue.enter(); + let workers = vec![WorkerId(0), WorkerId(1)]; + let directory = Arc::new( + WorkerDirectory::new( + Arc::new(WorkerMap::new(workers.clone()).unwrap()), + workers, + 4, + ) + .unwrap(), + ); + let page = page_owned_by(&directory, WorkerId(1)); + let full = uring_runtime::drivers::reserve().unwrap(); + { + let (local, _fixture) = crate::read::tests::admission_coordinator(); + let _local = directory.simulation_scope(Some((WorkerId(0), local))); + directory.report_sealed(failed_report(page.clone()), false, &scope()); + // A passing verdict is only a persistence hint and is not retained. + directory.report_sealed(failed_report(page), true, &scope()); + } + drop(full); + assert_eq!(applied_by_owner(&directory), 1); + assert!(directory.mailboxes[1].take_failed_verdicts().is_empty()); + } + #[test] + fn failed_sealed_verdict_survives_full_mailbox() { + let queue = Rc::new(uring_runtime::drivers::DriverQueue::new(64)); + let _queue = queue.enter(); + let workers = vec![WorkerId(0), WorkerId(1)]; + let directory = Arc::new( + WorkerDirectory::new( + Arc::new(WorkerMap::new(workers.clone()).unwrap()), + workers, + 1, + ) + .unwrap(), + ); + directory.mailboxes[1].install().unwrap(); + let page = page_owned_by(&directory, WorkerId(1)); + { + let (local, _fixture) = crate::read::tests::admission_coordinator(); + let _local = directory.simulation_scope(Some((WorkerId(0), local))); + // The first verdict takes the only mailbox slot; the second is retained. + directory.report_sealed(failed_report(page.clone()), false, &scope()); + assert_eq!(directory.mailboxes[1].outstanding(), 1); + directory.report_sealed(failed_report(page), false, &scope()); + } + assert_eq!(directory.mailboxes[1].take_failed_verdicts().len(), 1); + } + #[test] + fn failed_landed_verdict_settles_locally() { + let queue = Rc::new(uring_runtime::drivers::DriverQueue::new(64)); + let _queue = queue.enter(); + let workers = vec![WorkerId(0), WorkerId(1)]; + let directory = Arc::new( + WorkerDirectory::new( + Arc::new(WorkerMap::new(workers.clone()).unwrap()), + workers, + 4, + ) + .unwrap(), + ); + directory.mailboxes[1].install().unwrap(); + let page = page_owned_by(&directory, WorkerId(1)); + let (local, _fixture) = crate::read::tests::admission_coordinator(); + let before = local + .fill + .metrics + .count(crate::telemetry::Event::CorruptMiss); + { + let _local = directory.simulation_scope(Some((WorkerId(0), local.clone()))); + directory.report_sealed(landed_report(page), false, &scope()); + } + assert_eq!( + local + .fill + .metrics + .count(crate::telemetry::Event::CorruptMiss) + - before, + 1 + ); + assert_eq!(directory.mailboxes[1].outstanding(), 0); + assert!(directory.mailboxes[1].take_failed_verdicts().is_empty()); + } + #[test] + fn failed_sealed_verdicts_are_bounded() { + let directory = directory(1); + let page = page_owned_by(&directory, WorkerId(1)); + for _ in 0..FAILED_VERDICT_LIMIT { + assert!(directory.mailboxes[1].retain_failed_verdict(failed_report(page.clone()))); + } + assert!(!directory.mailboxes[1].retain_failed_verdict(failed_report(page))); + assert_eq!( + directory.mailboxes[1].take_failed_verdicts().len(), + FAILED_VERDICT_LIMIT + ); + } #[test] fn canonical_order_rejects_empty_and_duplicate_identities() { assert!(matches!( diff --git a/cmd/racer-dataplane/src/read/fill.rs b/cmd/racer-dataplane/src/read/fill.rs index 267ae5dacb..55ec6b1409 100644 --- a/cmd/racer-dataplane/src/read/fill.rs +++ b/cmd/racer-dataplane/src/read/fill.rs @@ -77,6 +77,18 @@ type LocalPlaintextKeys = std::cell::RefCell< std::collections::BTreeMap>>>, >; +/// A host descriptor to hand to cuFile, or None for simulated descriptors. +fn host_descriptor( + descriptor: &uring_runtime::reactor::descriptor::Descriptor, +) -> Option { + use std::os::fd::AsFd; + #[cfg(test)] + if descriptor.as_sim().is_some() { + return None; + } + descriptor.as_fd().try_clone_to_owned().ok() +} + /// Optional writes may use half of the aggregate class budget, with a floor /// sufficient for one valid page when the configured budget can hold it. Owned /// usage counts against this conservative ceiling; quota admission remains the @@ -889,6 +901,324 @@ impl Fill { } } + /// Acquire one page for direct GPU delivery: verified plaintext when a + /// reader still holds it on the CPU, otherwise ciphertext (from memory + /// first, then disk, peers or origin) for the GPU to verify and + /// decrypt. Network and disk copies keep the usual single-flight sharing, + /// memory publication and disk persistence; only the CPU decrypt and the + /// plaintext publication are skipped. + pub(crate) async fn acquire_sealed( + &self, + page: PageId, + membership: Arc, + context: &OriginContext, + scope: &RequestScope, + budget: &mut AcquisitionBudget, + target: &super::sealed::SealedTarget, + ) -> Result { + use super::sealed::Fetched; + // Verified memory, and pages another reader already authenticates, + // keep the plaintext path: the bytes are on the CPU anyway. + if self.cached_page(&page, scope)?.is_some() { + return self + .acquire(page, membership, context, scope, budget) + .await + .map(Fetched::Page); + } + // The memory cache keeps only ciphertext once readers drop the + // plaintext. A cached copy skips the disk and peer paths and is sealed + // for the GPU from host memory below. + let cached = self.dependencies.memory.unverified(&page)?.is_some(); + if !cached + && target.helper.cufile() + && self + .dependencies + .writer + .copy_only(&page) + .ok() + .flatten() + .is_none() + { + match self.stage_from_disk(&page, scope, target).await { + Ok(Some(sealed)) => return Ok(Fetched::Sealed(sealed)), + Ok(None) => {} + Err(error @ (Error::Cancelled | Error::DeadlineExceeded)) => return Err(error), + // Any other failure reads the page through host memory. + Err(_) => {} + } + } + if !cached { + match self + .land_from_peer(&page, membership.clone(), context, scope, budget, target) + .await + { + Ok(Some(fetched)) => return Ok(fetched), + Ok(None) => {} + Err(error @ (Error::Cancelled | Error::DeadlineExceeded)) => return Err(error), + // Any other failure takes the usual path through host memory. + Err(_) => {} + } + } + let acquired = self + .acquire_with_prefetch( + page.clone(), + membership.clone(), + context, + scope, + budget, + None, + false, + true, + ) + .await?; + match acquired { + AcquiredPage::Plaintext(result) => Ok(Fetched::Page(result)), + AcquiredPage::Ciphertext(copy) => match self.seal(&page, copy) { + Ok(sealed) => Ok(Fetched::Sealed(sealed)), + // A key the GPU cannot use, or an AAD too long for it: the + // CPU authenticates the page as for a Get. + Err(_) => self + .acquire(page, membership, context, scope, budget) + .await + .map(Fetched::Page), + }, + } + } + + /// Fetch a page from a peer straight into a GPU landing buffer (GPUDirect + /// RDMA). This is a private fetch for one sealed read: it does not join + /// or create a shared single flight, and a landed page is never published + /// to memory or persisted, because there are no host bytes to share. + /// + /// Only the first provider in visit order is asked, with one CopyOnly + /// request. None means use the usual path: no landing buffer on the + /// request's GPU, a local copy exists, this node owns the page, or the + /// provider missed or failed. A page the provider sent to host memory + /// instead (no free landing slot, or HTTP) is used privately in the same + /// way, so it is not fetched twice. + async fn land_from_peer( + &self, + page: &PageId, + membership: Arc, + context: &OriginContext, + scope: &RequestScope, + budget: &mut AcquisitionBudget, + target: &super::sealed::SealedTarget, + ) -> Result> { + let resolver = &self.dependencies.candidates; + if !resolver.landing_ready(target.ordinal) + || self + .dependencies + .writer + .copy_only(page) + .ok() + .flatten() + .is_some() + { + return Ok(None); + } + if let Some(local) = self.acquire_local_copy(page, scope, false).await? { + return Ok(Some(match local { + AcquiredPage::Plaintext(result) => super::sealed::Fetched::Page(result), + AcquiredPage::Ciphertext(copy) => match self.seal(page, copy) { + Ok(sealed) => super::sealed::Fetched::Sealed(sealed), + Err(_) => return Ok(None), + }, + })); + } + let candidates = resolver + .candidates_scoped(membership.clone(), &page.version.object, page.number, scope) + .await?; + if resolver.is_candidate(&candidates) { + return Ok(None); + } + let order = resolver.visit_order(&candidates, &page.version.object, page.number); + let Some(first) = order.ranks().next() else { + return Ok(None); + }; + let destination = candidates.ordered[first].clone(); + let delivered = resolver + .request_landing( + &membership, + &destination, + context, + page, + scope, + budget, + target.ordinal, + ) + .await?; + let landed = match delivered { + crate::peer::Delivered::Landed(landed) => landed, + // The provider sent the page to host memory (no free landing + // slot, or HTTP): use it privately rather than fetch it again. + crate::peer::Delivered::Response(response) => { + let Ok(copy) = response_copy(response.response(), page) else { + return Ok(None); + }; + return Ok(self + .seal( + page, + UnverifiedPage { + copy, + disk_token: None, + }, + ) + .ok() + .map(super::sealed::Fetched::Sealed)); + } + }; + let crate::peer::transport::LandedResponse { + metadata, + envelope, + landed, + .. + } = *landed; + if landed.ordinal() != target.ordinal || envelope.page != *page { + return Ok(None); + } + metadata.immutable().validate_page(&envelope)?; + let sealed = Box::new(super::sealed::SealedPage { + aad: crate::security::page_aad(&envelope)?, + key: self.dependencies.crypto.derive_page_key(&envelope)?, + // No host bytes to checksum: the GPU checks the AEAD tag. + expected_crc: None, + metadata, + envelope, + source: super::sealed::SealedSource::Landed(landed), + }); + sealed.validate_for(page)?; + self.metrics.record(Event::PeerHit, 1); + Ok(Some(super::sealed::Fetched::Sealed(sealed))) + } + + /// Package host ciphertext for GPU verification. + fn seal(&self, page: &PageId, copy: UnverifiedPage) -> Result> { + validate_copy(©.copy, page)?; + let envelope = copy.copy.ciphertext.envelope().clone(); + let sealed = Box::new(super::sealed::SealedPage { + metadata: copy.copy.metadata.clone(), + aad: crate::security::page_aad(&envelope)?, + key: self.dependencies.crypto.derive_page_key(&envelope)?, + expected_crc: copy.copy.ciphertext.cached_checksum(), + envelope, + source: super::sealed::SealedSource::Host(copy), + }); + sealed.validate_for(page)?; + Ok(sealed) + } + + /// Read a local v2 record's ciphertext straight into GPU staging with + /// cuFile. Only the header passes through host memory. None means the + /// page must be read through host memory: a miss, a v1 or unaligned + /// record, no free staging slot, or cuFile unavailable. + async fn stage_from_disk( + &self, + page: &PageId, + scope: &RequestScope, + target: &super::sealed::SealedTarget, + ) -> Result>> { + if target.helper.free_slots(target.ordinal) == 0 { + target + .helper + .counters() + .exhausted + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + return Ok(None); + } + let Some(record) = self + .dependencies + .disk + .direct_read(page, scope, |amount| { + self.reserve_with_reclamation( + &page.version.object.cache, + ResourceClass::Ciphertext, + amount, + ) + }) + .await? + else { + return Ok(None); + }; + if !record.ciphertext.aligned { + return Ok(None); + } + let Some(fd) = host_descriptor(&record.ciphertext.descriptor) else { + return Ok(None); + }; + let envelope = record.envelope.clone(); + if record.ciphertext.length != envelope.ciphertext_length as usize { + return Ok(None); + } + let aad = crate::security::page_aad(&envelope)?; + let key = self.dependencies.crypto.derive_page_key(&envelope)?; + let read_length = record.read_length; + let staged = target + .helper + .stage_file( + target.ordinal, + fd, + record.ciphertext.offset, + read_length as u64, + scope, + ) + .await; + // The read finished (or was abandoned): the bytes may now be recycled. + let crate::store::DirectRecord { + token, + metadata, + checksum, + lease, + .. + } = record; + drop(lease); + let staged = match staged { + Ok(staged) => staged, + Err(error @ (Error::Cancelled | Error::DeadlineExceeded)) => return Err(error), + Err(_) => return Ok(None), + }; + let sealed = Box::new(super::sealed::SealedPage { + metadata, + envelope, + aad, + key, + expected_crc: Some(checksum), + source: super::sealed::SealedSource::Staged { staged, token }, + }); + sealed.validate_for(page)?; + self.metrics.record(Event::DiskHit, 1); + self.metrics.record(Event::GpuDirectDiskRead, 1); + self.metrics + .record(Event::GpuDirectDiskReadBytes, read_length as u64); + Ok(Some(sealed)) + } + + /// Act on the GPU's verdict for a sealed page this worker supplied: a + /// rejected copy is dropped from memory and disk, as a failed CPU decrypt + /// would drop it; an authenticated network copy may now be persisted. + pub(crate) fn sealed_verdict(&self, report: super::sealed::SealedReport, passed: bool) { + let super::sealed::SealedReport { page, copy, token } = report; + if passed { + if let Some(copy) = copy + && copy.disk_token.is_none() + { + let observation = self.dependencies.writer.retention().owned_only(&page); + self.persist_copy(copy.copy, observation); + } + return; + } + self.metrics.record(Event::CorruptMiss, 1); + if let Some(copy) = © { + self.dependencies.memory.invalidate_ciphertext(copy); + if let Some(token) = ©.disk_token { + let _ = self.dependencies.disk.invalidate(token); + } + } + if let Some(token) = &token { + let _ = self.dependencies.disk.invalidate(token); + } + } + /// Warm gates own transfer exclusion until accepted acquisition work is fenced. pub(crate) async fn acquire_warm( &self, @@ -2113,7 +2443,20 @@ impl Fill { if !observation.eligible() || result.validate_metadata().is_err() { return; } - let id = result.plaintext.page(); + self.persist_copy(result.copy(), observation); + } + + /// Queue an authenticated copy for disk under the reader's latched + /// retention decision. The caller must have authenticated the bytes. + fn persist_copy( + &self, + copy: crate::memory::CiphertextCopy, + observation: crate::retention::Observation, + ) { + if !observation.eligible() || copy.validate_metadata().is_err() { + return; + } + let id = ©.ciphertext.envelope().page.clone(); if self .dependencies .writer @@ -2135,13 +2478,13 @@ impl Fill { { return; } - let cache = &result.metadata.version.object.cache; + let cache = ©.metadata.version.object.cache.clone(); let retention = self.dependencies.writer.retention(); // Ownership is current here, but second-sight eligibility remains the // latched observation. Reclassification cannot self-promote Bloom state. let owned = retention.owned_only(id).eligible(); let admission = &self.dependencies.admission; - let amount = result.ciphertext.bytes().len(); + let amount = copy.ciphertext.bytes().len(); // Optional pages of an over-share cache do not displace other caches' // data once free segments reach the reserve. Owned pages compete // through eviction instead. @@ -2166,7 +2509,7 @@ impl Fill { .reserve(Some(cache), ResourceClass::DirtyCiphertext, amount) .map_err(Error::from) }; - let page_charge = result + let page_charge = copy .ciphertext .inner .reservation() @@ -2187,7 +2530,7 @@ impl Fill { let result = self .dependencies .writer - .enqueue_reclaiming(result.copy(), dirty, |amount| { + .enqueue_reclaiming(copy, dirty, |amount| { // Ciphertext owns both the immutable page and its disk staging. // The one-page floor includes BOTH, even on tiny slabs. The page // term is the page's own Ciphertext charge, which for aligned diff --git a/cmd/racer-dataplane/src/read/range_stream.rs b/cmd/racer-dataplane/src/read/range_stream.rs index d0431e5bbe..59b358f8c3 100644 --- a/cmd/racer-dataplane/src/read/range_stream.rs +++ b/cmd/racer-dataplane/src/read/range_stream.rs @@ -23,6 +23,10 @@ use crate::model::PageNumber; use crate::model::ResolvedRange; #[cfg(test)] use crate::read::dispatch::WorkerMap; +use crate::read::sealed::Fetched; +use crate::read::sealed::SealedPage; +use crate::read::sealed::SealedReport; +use crate::read::sealed::SealedTarget; use crate::runtime::RequestScope; use crate::security::OriginContext; use crate::store::holds::HoldEpoch; @@ -40,7 +44,8 @@ use std::task::Waker; use std::time::Duration; use std::time::Instant; -/// Direct fills a read keeps in flight beyond its head page. +/// Direct fills a read keeps in flight beyond its head page. A sealed read +/// fills its whole window directly (see `RangeStream::direct_ahead`). const DIRECT_AHEAD: usize = 1; /// Provider selections a read keeps in flight. Beyond the first, a read only @@ -478,6 +483,16 @@ impl RangeBudget { type Fetch = Operation<'static, (Result, AcquisitionBudget)>; +/// A direct fill in sealed mode, which may return ciphertext for the GPU. +type SealedFetch = Operation<'static, (Result, AcquisitionBudget)>; + +/// One page for a direct read: plaintext, or ciphertext the sink verifies. +pub(crate) enum DirectItem { + Plaintext(ReaderLease), + /// A sealed page and the slice of it the range covers. + Sealed(Box, crate::model::PageSlice), +} + /// One page of the read window, `slots[i]` is page `next + i`. #[allow(clippy::large_enum_variant)] // Avoid allocating again when an in-window page becomes ready. enum Slot { @@ -488,6 +503,10 @@ enum Slot { Missing { select: bool }, /// A direct fill through the owning worker. Fetching(Fetch), + /// A direct fill in sealed mode. + FetchingSealed(SealedFetch), + /// Fetched as ciphertext for the GPU to verify. + Sealed(Box), /// Covered by an outstanding provider selection. Selecting, /// Fetched, or failed. An error waits here until the page reaches the head. @@ -576,6 +595,9 @@ pub struct RangeStream { spill: Option, + /// Set for direct GPU reads: misses are fetched sealed for this GPU. + sealed: Option, + terminated: bool, } @@ -593,6 +615,11 @@ impl RangeStreams { } } + /// The configured `range_window_pages` of each stream. + pub(crate) fn window_pages(&self) -> usize { + self.window_pages + } + /// Share the same metadata and page owners used by newly opened streams. pub(crate) fn directory(&self) -> &Arc { &self.directory @@ -640,6 +667,7 @@ impl RangeStreams { slots: VecDeque::new(), selections: Vec::new(), spill: None, + sealed: None, terminated: false, }) } @@ -678,6 +706,21 @@ impl RangeStream { Ok(()) } + /// Fetch pages that are not verified plaintext in memory as sealed pages + /// for `target`'s GPU. Call before the first delivery. + pub(crate) fn set_sealed(&mut self, target: SealedTarget) -> Result<()> { + if self.terminated || !self.slots.is_empty() { + return Err(Error::InvalidRequest); + } + self.sealed = Some(target); + Ok(()) + } + + /// Tell a sealed page's owner what the GPU decided. + pub(crate) fn report_sealed(&self, report: SealedReport, passed: bool) { + self.directory.report_sealed(report, passed, &self.scope); + } + /// Release exactly one delivered page and its retained plaintext charge. pub fn release_page(&mut self, number: PageNumber, length: u32) -> Result<()> { self.credits @@ -739,11 +782,18 @@ impl RangeStream { }; let directory = self.directory.clone(); let scope = scope.clone(); + let sealed = self.sealed.is_some(); let spilled = self .spill .as_mut() .and_then(|spill| spill.pages.remove(&number.0)); self.slots.push_back(Slot::Probing(match spilled { + // A sealed read probes live plaintext only. Cached or disk + // ciphertext is then fetched sealed, so the GPU rather than + // the CPU decrypts it. + None if sealed => { + Box::pin(async move { directory.cached_page(page, &scope).await }) + } // Holding pages stay with the spill state until the hold // completes, so the hold is released exactly once. None => Box::pin(async move { directory.local_page(page, &scope).await }), @@ -753,8 +803,21 @@ impl RangeStream { .unwrap() .pages .insert(number.0, Spilling::Holding(holding)); - Box::pin(async move { directory.local_page(page, &scope).await }) + if sealed { + Box::pin(async move { directory.cached_page(page, &scope).await }) + } else { + Box::pin(async move { directory.local_page(page, &scope).await }) + } + } + // A sealed read probes memory only: a spilled page on disk is + // then fetched sealed, so the GPU decrypts it. + Some(Spilling::Spilled) if sealed => { + Box::pin(async move { directory.cached_page(page, &scope).await }) } + Some(Spilling::Warming(warming)) if sealed => Box::pin(async move { + let _ = warming.await; + directory.cached_page(page, &scope).await + }), // A spilled page is read back from memory or its held disk copy. // If that fails the page is fetched normally. Some(Spilling::Spilled) => { @@ -828,6 +891,28 @@ impl RangeStream { } Poll::Ready(Err(error)) => Slot::Ready(Err(error)), }, + Slot::FetchingSealed(fetch) => match fetch.as_mut().poll(cx) { + Poll::Pending => continue, + Poll::Ready(Ok((result, remaining))) => { + let completed = self.budget.complete(remaining); + match result { + Ok(Fetched::Page(page)) => { + if let Some(lease) = &self.lease { + lease.offer(&page); + } + Slot::Ready(completed.map(|()| page)) + } + Ok(Fetched::Sealed(sealed)) => match completed + .and_then(|()| self.validate_sealed(&sealed, number)) + { + Ok(()) => Slot::Sealed(sealed), + Err(error) => Slot::Ready(Err(error)), + }, + Err(error) => Slot::Ready(completed.and(Err(error))), + } + } + Poll::Ready(Err(error)) => Slot::Ready(Err(error)), + }, _ => continue, }; self.slots[index] = next; @@ -903,10 +988,10 @@ impl RangeStream { || self .slots .iter() - .any(|slot| matches!(slot, Slot::Fetching(_))) + .any(|slot| matches!(slot, Slot::Fetching(_) | Slot::FetchingSealed(_))) } - /// Start a direct fill for the head page, up to `DIRECT_AHEAD` more holes, + /// Start a direct fill for the head page, up to `direct_ahead` more holes, /// and provider selections for the remaining holes. fn admit(&mut self, scope: &RequestScope) -> Result { // A failed page stops admission at and after it until it reaches the @@ -933,7 +1018,7 @@ impl RangeStream { .slots .iter() .skip(1) - .filter(|slot| matches!(slot, Slot::Fetching(_))) + .filter(|slot| matches!(slot, Slot::Fetching(_) | Slot::FetchingSealed(_))) .count(); // Pages another window is selecting or filling arrive through its // fan-out. Only the head is always filled directly, so it never waits @@ -945,7 +1030,8 @@ impl RangeStream { .map(WindowLease::busy_elsewhere) .unwrap_or_default(); let mut index = 1; - while ahead < DIRECT_AHEAD && index < limit { + let direct_ahead = self.direct_ahead(); + while ahead < direct_ahead && index < limit { if matches!(self.slots[index], Slot::Missing { .. }) && !elsewhere.contains(&self.page_at(index)) { @@ -965,6 +1051,18 @@ impl RangeStream { Ok(changed) } + /// Direct fills to keep in flight beyond the head page. Provider selections + /// deliver verified plaintext, so a sealed read fills its whole window + /// directly. Its direct fills stage disk pages with cuFile and keep + /// ciphertext for the GPU to decrypt. + fn direct_ahead(&self) -> usize { + if self.sealed.is_some() { + self.window_pages.saturating_sub(1).max(DIRECT_AHEAD) + } else { + DIRECT_AHEAD + } + } + /// Returns false when the range budget admits no more work for now. fn start_fetch(&mut self, index: usize, scope: &RequestScope) -> Result { let Some(child) = self.budget.next_page(self.pending())? else { @@ -972,20 +1070,34 @@ impl RangeStream { }; let mut child_scope = scope.clone(); child_scope.deadline.0 = child.deadline(); - let fetch = self.directory.start_page( - PageId { - version: self.metadata.version.clone(), - number: PageNumber(self.page_at(index)), - }, - self.membership.clone(), - &self.context, - &child_scope, - child, - ); - self.slots[index] = match fetch { - Ok(fetch) => Slot::Fetching(fetch), - Err(error) => Slot::Ready(Err(error)), + let page = PageId { + version: self.metadata.version.clone(), + number: PageNumber(self.page_at(index)), }; + let fetch = match &self.sealed { + Some(target) => self + .directory + .start_page_sealed( + page, + self.membership.clone(), + &self.context, + &child_scope, + child, + target.clone(), + ) + .map(Slot::FetchingSealed), + None => self + .directory + .start_page( + page, + self.membership.clone(), + &self.context, + &child_scope, + child, + ) + .map(Slot::Fetching), + }; + self.slots[index] = fetch.unwrap_or_else(|error| Slot::Ready(Err(error))); Ok(true) } @@ -1139,7 +1251,7 @@ impl RangeStream { wants.insert(number); busy.insert(number); } - Slot::Fetching(_) => { + Slot::Fetching(_) | Slot::FetchingSealed(_) => { busy.insert(number); } Slot::Probing(_) | Slot::Missing { .. } => { @@ -1337,6 +1449,13 @@ impl RangeStream { } scope } + fn validate_sealed(&self, sealed: &SealedPage, number: PageNumber) -> Result<()> { + validate_pin(&self.metadata, &sealed.metadata)?; + sealed.validate_for(&PageId { + version: self.metadata.version.clone(), + number, + }) + } fn validate(&self, result: &PageResult, number: PageNumber) -> Result<()> { validate_pin(&self.metadata, &result.metadata)?; result.validate_for(&PageId { @@ -1347,6 +1466,22 @@ impl RangeStream { /// Deliver the next page in order. Pending futures live in the stream, so /// dropping next_slice cannot restart a page or refill its retries. pub fn next_slice(&mut self) -> Operation<'_, Option> { + Box::pin(async move { + match self.next_direct().await? { + None => Ok(None), + Some(DirectItem::Plaintext(reader)) => Ok(Some(reader)), + // Only direct GPU reads set a sealed target. + Some(DirectItem::Sealed(..)) => { + self.terminate(); + Err(Error::Internal) + } + } + }) + } + + /// Deliver the next page in order for a direct read, sealed if the + /// stream has a sealed target and the page was fetched as ciphertext. + pub(crate) fn next_direct(&mut self) -> Operation<'_, Option> { Box::pin(async move { if self.terminated { return Ok(None); @@ -1384,10 +1519,10 @@ impl RangeStream { /// Whether the head page can be handed to the client now. fn deliverable(&self) -> bool { - matches!(self.slots.front(), Some(Slot::Ready(_))) + matches!(self.slots.front(), Some(Slot::Ready(_) | Slot::Sealed(_))) } - async fn deliver(&mut self) -> Result> { + async fn deliver(&mut self) -> Result> { let scope = self.operation_scope(); let cancellation = scope.cancellation.subscribe()?; poll_fn(|cx| -> Poll> { @@ -1408,20 +1543,37 @@ impl RangeStream { return Ok(None); } scope.check()?; - let Some(Slot::Ready(result)) = self.slots.pop_front() else { - return Err(Error::StaleFlight); - }; let number = PageNumber(self.next); - self.next += 1; - let result = result?; + let result = match self.slots.pop_front() { + Some(Slot::Ready(result)) => { + self.next += 1; + result? + } + Some(Slot::Sealed(sealed)) => { + self.next += 1; + self.validate_sealed(&sealed, number)?; + let slice = self.range.slice_at(number)?.ok_or(Error::CorruptRecord)?; + self.issue(number)?; + return Ok(Some(DirectItem::Sealed(sealed, slice))); + } + _ => return Err(Error::StaleFlight), + }; self.validate(&result, number)?; let slice = self.range.slice_at(number)?.ok_or(Error::CorruptRecord)?; + self.issue(number)?; + self.retained.insert(number, result.plaintext.clone()); + self.delivery + .attach(result.plaintext, slice) + .map(|reader| Some(DirectItem::Plaintext(reader))) + } + + /// Account one page as handed to the client, and drop its spill hold. + fn issue(&mut self, number: PageNumber) -> Result<()> { self.credits .as_mut() .ok_or(Error::InvalidRequest)? .issued(number) .map_err(window_error)?; - self.retained.insert(number, result.plaintext.clone()); if let Some(spill) = self.spill.as_mut() && let Some(epoch) = spill.held.remove(&number.0) { @@ -1431,14 +1583,23 @@ impl RangeStream { self.scope.request, ); } - self.delivery.attach(result.plaintext, slice).map(Some) + Ok(()) } /// Pages admitted into the window: fetched, being fetched or selected. pub fn buffered_pages(&self) -> usize { self.slots .iter() - .filter(|slot| matches!(slot, Slot::Fetching(_) | Slot::Selecting | Slot::Ready(_))) + .filter(|slot| { + matches!( + slot, + Slot::Fetching(_) + | Slot::FetchingSealed(_) + | Slot::Selecting + | Slot::Ready(_) + | Slot::Sealed(_) + ) + }) .count() } } @@ -1596,6 +1757,92 @@ pub(super) mod tests { } } + #[test] + fn sealed_reads_fill_their_whole_window_directly() { + for (window, sealed) in [(1, 1), (2, 1), (8, 7)] { + let f = Fixture::new(crate::test_support::cluster::config(false).limits, window); + let mut metadata = metadata(); + metadata.length = 16 * PAGE_BYTES; + let range = ByteRange::From(0).resolve(metadata.length).unwrap(); + let scope = + RequestScope::new(RequestId([9; 16]), Instant::now() + Duration::from_secs(60)) + .unwrap(); + let mut stream = f.open(&metadata, range, scope); + assert_eq!(stream.direct_ahead(), DIRECT_AHEAD); + stream + .set_sealed(SealedTarget { + helper: crate::gpu::testing::helper(&[None]), + ordinal: 0, + }) + .unwrap(); + assert_eq!(stream.direct_ahead(), sealed, "window {window}"); + } + } + + #[test] + fn sealed_pages_are_delivered_in_order_only_to_direct_readers() { + let f = Fixture::new(crate::test_support::cluster::config(false).limits, 2); + let admission = &f.admission; + let mut metadata = metadata(); + metadata.length = 2 * PAGE_BYTES; + let range = ByteRange::Closed { + first: 5, + last: PAGE_BYTES + 9, + } + .resolve(metadata.length) + .unwrap(); + let scope = || { + RequestScope::new(RequestId([9; 16]), Instant::now() + Duration::from_secs(60)).unwrap() + }; + let mut stream = f.open(&metadata, range, scope()); + stream.configure_subscription(4, 4 * PAGE_BYTES).unwrap(); + let first = crate::read::sealed::tests::sealed(&page_result(admission, &metadata, 0)); + stream.inject(Slot::Sealed(Box::new(first))); + stream.inject(Slot::Ready(Ok(page_result(admission, &metadata, 1)))); + match futures::executor::block_on(stream.next_direct()).unwrap() { + Some(DirectItem::Sealed(page, slice)) => { + assert_eq!(page.page().number, PageNumber(0)); + assert_eq!( + (slice.page, slice.offset, slice.length), + (PageNumber(0), 5, PAGE_BYTES as u32 - 5) + ); + } + _ => panic!("expected a sealed page"), + } + stream + .release_page(PageNumber(0), PAGE_BYTES as u32 - 5) + .unwrap(); + match futures::executor::block_on(stream.next_direct()).unwrap() { + Some(DirectItem::Plaintext(reader)) => { + assert_eq!(reader.slice().page, PageNumber(1)); + assert_eq!(reader.slice().length, 10); + } + _ => panic!("expected plaintext"), + } + assert!( + futures::executor::block_on(stream.next_direct()) + .unwrap() + .is_none() + ); + + // A sealed page for another page number fails that page. + let mut stream = f.open(&metadata, range, scope()); + stream.configure_subscription(4, 4 * PAGE_BYTES).unwrap(); + let other = crate::read::sealed::tests::sealed(&page_result(admission, &metadata, 1)); + stream.inject(Slot::Sealed(Box::new(other))); + assert!(futures::executor::block_on(stream.next_direct()).is_err()); + + // A Get stream never yields sealed pages. + let mut stream = f.open(&metadata, range, scope()); + stream.configure_subscription(4, 4 * PAGE_BYTES).unwrap(); + let first = crate::read::sealed::tests::sealed(&page_result(admission, &metadata, 0)); + stream.inject(Slot::Sealed(Box::new(first))); + assert_eq!( + futures::executor::block_on(stream.next_slice()).err(), + Some(Error::Internal) + ); + } + #[test] fn retained_boundary_slices_keep_full_plaintext_charged_until_exact_release() { let f = Fixture::new(crate::test_support::cluster::config(false).limits, 2); diff --git a/cmd/racer-dataplane/src/read/sealed.rs b/cmd/racer-dataplane/src/read/sealed.rs new file mode 100644 index 0000000000..6943d078f7 --- /dev/null +++ b/cmd/racer-dataplane/src/read/sealed.rs @@ -0,0 +1,230 @@ +//! Sealed pages: ciphertext a GPU authenticates and decrypts itself. +//! +//! A `gpu-read` asks its range stream for sealed pages +//! ([`super::range_stream::RangeStream::set_sealed`]). Pages that are verified +//! plaintext in memory still arrive as [`Fetched::Page`], and so do pages that +//! the CPU had to produce anyway (origin fills, which encrypt) or that another +//! reader authenticated first. Every other page arrives as a [`SealedPage`]: +//! the envelope, the AAD, the expected CRC, the XAES-derived page key, and the +//! ciphertext in host memory or already in GPU staging. +//! +//! The derived key is wiped on drop and is never logged: [`SealedPage`] has no +//! `Debug` implementation. + +use crate::error::Error; +use crate::error::Result; +use crate::gpu::GpuHelper; +use crate::gpu::LandedCiphertext; +use crate::gpu::StagedCiphertext; +use crate::memory::PageResult; +use crate::memory::UnverifiedPage; +use crate::model::ObjectMetadata; +use crate::model::PageEnvelope; +use crate::model::PageId; +use crate::store::ReadToken; +use std::sync::Arc; + +/// Where a GPU should read a sealed page from. +#[derive(Clone)] +pub struct SealedTarget { + pub(crate) helper: Arc, + + /// The GPU that holds the request's allocation. + pub(crate) ordinal: i32, +} + +impl SealedTarget { + /// The GPU that holds the request's allocation. + pub fn ordinal(&self) -> i32 { + self.ordinal + } +} + +/// One fetched window page in sealed mode. +#[allow(clippy::large_enum_variant)] // Pages move unboxed through the mailbox, as `Value::Page` does. +pub enum Fetched { + /// Verified plaintext. + Page(PageResult), + + /// Ciphertext for the GPU to verify and decrypt. + Sealed(Box), +} + +/// Where a sealed page's ciphertext is. +#[allow(clippy::large_enum_variant)] // Always inside a boxed SealedPage. +pub enum SealedSource { + /// Ciphertext and tag in host memory: retained, pending, read from disk + /// through the slab, or received from a peer. + Host(UnverifiedPage), + + /// Ciphertext already in the target GPU's staging memory, read from a + /// local v2 disk record with cuFile. + Staged { + staged: StagedCiphertext, + + /// The disk mapping the bytes came from. + token: ReadToken, + }, + + /// Ciphertext a peer's NIC wrote straight into the target GPU's RDMA + /// landing buffer. No host copy exists, so nothing is shared, published, + /// or persisted, and the page is used once. + Landed(LandedCiphertext), +} + +/// A page that only the GPU will authenticate. Nothing here is proof of +/// authenticity: the GPU checks the CRC and the AEAD tag before it writes. +pub struct SealedPage { + pub metadata: ObjectMetadata, + + pub envelope: PageEnvelope, + + /// `security::page_aad(&envelope)`. + pub aad: Vec, + + /// The XAES-derived AES-256-GCM key and nonce. Wiped on drop. + pub key: racer_crypto::DerivedPageKey, + + /// The stored CRC-64 of ciphertext and tag, when known. + pub expected_crc: Option, + + pub source: SealedSource, +} + +/// What a page's owning worker needs to act on the GPU's verdict. +#[derive(Clone)] +pub struct SealedReport { + pub(crate) page: PageId, + + /// The host copy, to drop from memory on failure or persist on success. + pub(crate) copy: Option, + + /// The disk mapping a staged copy came from, to drop on failure. + pub(crate) token: Option, +} + +impl SealedReport { + /// Whether the owner holds anything to persist or invalidate for this + /// page. Landed pages are used once and leave nothing behind. + pub(crate) fn has_owner_state(&self) -> bool { + self.copy.is_some() || self.token.is_some() + } +} + +impl SealedPage { + /// Owner-side facts about where the bytes came from. Cheap: it shares + /// the ciphertext and copies no secrets. + pub fn report(&self) -> SealedReport { + let (copy, token) = match &self.source { + SealedSource::Host(copy) => (Some(copy.clone()), None), + SealedSource::Staged { token, .. } => (None, Some(token.clone())), + SealedSource::Landed(_) => (None, None), + }; + SealedReport { + page: self.page().clone(), + copy, + token, + } + } + + /// The page identity from the envelope. + pub fn page(&self) -> &PageId { + &self.envelope.page + } + + /// Plaintext bytes in the whole page. + pub fn plaintext_length(&self) -> u32 { + self.envelope.plaintext_length + } + + /// Structural checks only: the identity, lengths and AAD bound agree. + pub fn validate_for(&self, page: &PageId) -> Result<()> { + self.metadata.validate()?; + self.envelope.validate()?; + if self.page() != page + || self.metadata.version != page.version + || self.metadata.immutable().page_length(page)? != self.envelope.plaintext_length + || self.aad.len() > racer_gpu::MAX_AAD + { + return Err(Error::CorruptRecord); + } + if let SealedSource::Landed(landed) = &self.source + && landed.len() != u64::from(self.envelope.ciphertext_length) + { + return Err(Error::CorruptRecord); + } + if let SealedSource::Host(copy) = &self.source { + let ciphertext = ©.copy.ciphertext; + if ciphertext.envelope() != &self.envelope + || ciphertext.bytes().len() != self.envelope.ciphertext_length as usize + { + return Err(Error::CorruptRecord); + } + } + Ok(()) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use crate::memory::CiphertextCopy; + + /// A host-sourced sealed page for `result`, with a fixed test key. + pub(crate) fn sealed(result: &PageResult) -> SealedPage { + let envelope = result.ciphertext.envelope().clone(); + SealedPage { + metadata: result.metadata.clone(), + aad: crate::security::page_aad(&envelope).unwrap_or_default(), + key: racer_crypto::xaes_derive(&[7; 32], &envelope.nonce.0), + expected_crc: Some(result.ciphertext.checksum()), + envelope, + source: SealedSource::Host(UnverifiedPage { + copy: CiphertextCopy { + metadata: result.metadata.clone(), + ciphertext: result.ciphertext.clone(), + }, + disk_token: None, + }), + } + } + + #[test] + fn validate_checks_identity_lengths_and_aad_bound() { + use crate::model::*; + let admission = flow_control::Quotas::new(crate::admission::AdmissionPolicy::new( + crate::test_support::cluster::config(false).limits, + )); + let metadata = ObjectMetadata { + content_type: None, + version: ObjectVersion { + object: ObjectId { + cache: racer_control_wire::CacheId( + "00000000-0000-4000-8000-000000000001".into(), + ), + key: CacheKey([0; 32]), + }, + etag: StrongEtag::test_value("v1"), + }, + length: 37, + expires_at: ExpiresAt::from_system_time(std::time::UNIX_EPOCH).unwrap(), + }; + let result = crate::read::range_stream::tests::page_result(&admission, &metadata, 0); + let page = result.plaintext.page().clone(); + let mut sealed = sealed(&result); + sealed.validate_for(&page).unwrap(); + assert_eq!(sealed.plaintext_length(), 37); + + let mut other = page.clone(); + other.number = crate::model::PageNumber(1); + assert_eq!(sealed.validate_for(&other), Err(Error::CorruptRecord)); + + sealed.aad = vec![0; racer_gpu::MAX_AAD + 1]; + assert_eq!(sealed.validate_for(&page), Err(Error::CorruptRecord)); + + let mut sealed = self::sealed(&result); + sealed.envelope.plaintext_length += 1; + sealed.envelope.ciphertext_length += 1; + assert!(sealed.validate_for(&page).is_err()); + } +} diff --git a/cmd/racer-dataplane/src/read/tests/peer_copies.rs b/cmd/racer-dataplane/src/read/tests/peer_copies.rs index 3dcce9a8b5..4128c680c2 100644 --- a/cmd/racer-dataplane/src/read/tests/peer_copies.rs +++ b/cmd/racer-dataplane/src/read/tests/peer_copies.rs @@ -2970,3 +2970,245 @@ fn registered_copy_out_refused_by_admission_keeps_the_registered_page() { crate::rdma::links::tests::pump(&mut w.a.owner, &mut w.b.owner); assert_eq!(free_buffers(&w), Some(2)); } + +/// GPU landing for `gpu-read` (GPUDirect RDMA) through `Fill::acquire_sealed`. +mod landing { + use super::*; + use crate::gpu::GpuHelper; + use crate::read::sealed::Fetched; + use crate::read::sealed::SealedSource; + use crate::read::sealed::SealedTarget; + + struct Landing { + f: Fixture, + peers: Rc, + ordered: Vec, + helper: Arc, + copy: CiphertextCopy, + /// Requests the landing script saw. + asked: Rc>, + } + + /// A non-owner whose first provider may land page 0 on GPU `ordinal`. + /// With `lands`, the script answers with the real ciphertext written into + /// the landing buffer, as a NIC would; otherwise the request goes to the + /// usual peer script. + fn landing(ordinal: i32, lands: bool) -> Landing { + let mut f = fixture(); + let copy = encrypted_copy(&mut f); + let (peers, ordered) = install_peers(&mut f, None); + let helper = crate::gpu::testing::helper(&[None, None]); + let buffer = crate::gpu::testing::landing(&helper, 0, 2 * crate::gpu::SLOT_BYTES).unwrap(); + crate::gpu::testing::nic_write(&buffer, 0, copy.ciphertext.bytes()); + let asked = Rc::new(Cell::new(0)); + let envelope = copy.ciphertext.envelope().clone(); + let metadata = copy.metadata.clone(); + let length = copy.ciphertext.bytes().len() as u64; + let seen = asked.clone(); + let land: crate::peer::ScriptedLanding = Box::new(move |request| { + seen.set(seen.get() + 1); + assert!(matches!( + request.operation, + PeerOperation::Page { + mode: FetchMode::CopyOnly, + .. + } + )); + if !lands { + return None; + } + Some(crate::peer::transport::LandedResponse { + // The fill path never reads the head: the transport verified it. + authentication: crate::peer::forwarding::ForwardedHead { + original: Arc::new(peer_wire::SignedHead { + head: http1::MessageHead { + start: http1::StartLine::Request { + method: "POST".into(), + target: "/test".into(), + }, + headers: vec![], + }, + signature: vec![], + }), + hops: vec![], + }, + metadata: metadata.clone(), + envelope: envelope.clone(), + landed: crate::gpu::LandedCiphertext::for_test(&buffer, 0, length), + }) + }); + let mut dependencies = f.fill.dependencies.clone(); + dependencies.candidates = Rc::new(CandidatePolicy::new( + node(peers.local), + Rc::new(Placement::new(16)), + Requester::scripted( + peers.clone(), + ScriptedPeers::direct_hedge_available, + ScriptedPeers::request, + ScriptedPeers::request_direct, + ) + .with_scripted_landing(ordinal, land), + dependencies.credentials.clone(), + Arc::new(controlplane::Published::new( + crate::control::Snapshot::retention(2), + )), + )); + f.fill = Fill::new(dependencies); + f.fill + .dependencies + .writer + .retention() + .set_ownership(Rc::new(|_| false)); + Landing { + f, + peers, + ordered, + helper, + copy, + asked, + } + } + + fn acquire_sealed(l: &mut Landing) -> Result { + let target = SealedTarget { + helper: l.helper.clone(), + ordinal: 0, + }; + let mut budget = AcquisitionBudget::new(l.f.scope.deadline.0, 16, 32); + let f = &mut l.f; + drive( + f.fill.acquire_sealed( + f.page.clone(), + f.membership.clone(), + &f.context, + &f.scope, + &mut budget, + &target, + ), + &mut f.engine, + &f.crypto, + ) + } + + fn landed_count(helper: &GpuHelper) -> u64 { + helper + .counters() + .landed + .load(std::sync::atomic::Ordering::Relaxed) + } + + #[test] + /// A landed page decrypts on the GPU and is private to this read: no + /// CPU decrypt, no memory publication, no persistence, no host + /// ciphertext charge, and a later plain read fetches it again. + fn landed_peer_page_decrypts_on_the_gpu_and_stays_private() { + let queue = Rc::new(uring_runtime::drivers::DriverQueue::new(1024)); + let _owner = queue.enter(); + let mut l = landing(0, true); + let ciphertext = + l.f.fill + .dependencies + .admission + .used(ResourceClass::Ciphertext); + let Fetched::Sealed(sealed) = acquire_sealed(&mut l).unwrap() else { + panic!("expected a sealed page"); + }; + assert!(matches!(sealed.source, SealedSource::Landed(_))); + assert!(sealed.expected_crc.is_none()); + // One landing request to the first provider, none through the script. + assert_eq!(l.asked.get(), 1); + assert!(l.peers.calls.borrow().is_empty()); + assert_unpublished(&l.f); + assert_eq!( + l.f.fill + .dependencies + .admission + .used(ResourceClass::Ciphertext), + ciphertext + ); + let report = sealed.report(); + assert!(report.copy.is_none() && report.token.is_none()); + let fd = crate::gpu::testing::import(&l.helper, 1, 0); + crate::gpu::testing::write(&l.helper, 1, sealed, 0..3).unwrap(); + assert_eq!(racer_gpu::simulation::read_allocation(&fd, 0, 3), b"abc"); + assert_eq!(landed_count(&l.helper), 1); + // A passed verdict persists nothing: there is no host copy. + l.f.fill.sealed_verdict(report, true); + assert_unpublished(&l.f); + // Nothing was shared: a later plain read fetches the page again. + let deadline = l.f.scope.deadline.0; + l.peers + .replies + .borrow_mut() + .push_back((l.ordered[0].clone(), Reply::Copy(l.copy.clone()))); + let result = acquire(&mut l.f, &mut AcquisitionBudget::new(deadline, 16, 32)).unwrap(); + assert_eq!(result.plaintext.bytes(), b"abc"); + assert_eq!(l.peers.calls.borrow().len(), 1); + drop(fd); + assert_eq!(l.helper.leaked(), 0); + } + + #[test] + /// No free landing slot at this node: the provider sends the page to + /// host memory, and that answer is used once rather than fetched again. + fn host_answer_to_a_landing_request_is_used_once_and_not_published() { + let queue = Rc::new(uring_runtime::drivers::DriverQueue::new(1024)); + let _owner = queue.enter(); + let mut l = landing(0, false); + l.peers + .replies + .borrow_mut() + .push_back((l.ordered[0].clone(), Reply::Copy(l.copy.clone()))); + let Fetched::Sealed(sealed) = acquire_sealed(&mut l).unwrap() else { + panic!("expected a sealed page"); + }; + assert!(matches!(sealed.source, SealedSource::Host(_))); + assert_eq!(l.asked.get(), 1); + assert_eq!(l.peers.calls.borrow().len(), 1, "not fetched twice"); + assert!( + l.f.fill + .dependencies + .memory + .get(&l.f.page) + .unwrap() + .is_none() + ); + let fd = crate::gpu::testing::import(&l.helper, 1, 0); + crate::gpu::testing::write(&l.helper, 1, sealed, 0..3).unwrap(); + assert_eq!(racer_gpu::simulation::read_allocation(&fd, 0, 3), b"abc"); + assert_eq!(landed_count(&l.helper), 0); + } + + #[test] + /// A landing buffer on another GPU is never used, and a failed landing + /// request falls back to the usual acquisition. + fn landing_falls_back_to_the_usual_path() { + let queue = Rc::new(uring_runtime::drivers::DriverQueue::new(1024)); + let _owner = queue.enter(); + let mut l = landing(1, true); + l.peers + .replies + .borrow_mut() + .push_back((l.ordered[0].clone(), Reply::Copy(l.copy.clone()))); + let fetched = acquire_sealed(&mut l).unwrap(); + assert!(!matches!( + fetched, + Fetched::Sealed(ref sealed) if matches!(sealed.source, SealedSource::Landed(_)) + )); + assert_eq!(l.asked.get(), 0); + assert_eq!(l.peers.calls.borrow().len(), 1); + let mut l = landing(0, false); + { + let mut replies = l.peers.replies.borrow_mut(); + replies.push_back((l.ordered[0].clone(), Reply::Unauthorized)); + for node in &l.ordered { + replies.push_back((node.clone(), Reply::Copy(l.copy.clone()))); + } + } + let fetched = acquire_sealed(&mut l); + assert!(fetched.is_ok(), "{:?}", fetched.err()); + assert_eq!(l.asked.get(), 1); + assert!(l.peers.calls.borrow().len() >= 2); + assert_eq!(landed_count(&l.helper), 0); + } +} diff --git a/cmd/racer-dataplane/src/security.rs b/cmd/racer-dataplane/src/security.rs index ff3f09493e..627ae7171c 100644 --- a/cmd/racer-dataplane/src/security.rs +++ b/cmd/racer-dataplane/src/security.rs @@ -317,6 +317,22 @@ impl PageCrypto { self.buffers = Some(buffers); self } + /// Derive the AES-256-GCM key and nonce that XAES-256-GCM uses for this + /// page, so a GPU can authenticate and decrypt it. Checks the cache, key ID + /// and purpose exactly as [`Self::decrypt`] does. The derived key is valid + /// for this one page nonce and is wiped on drop. Three AES blocks, so it + /// runs on the I/O thread rather than queueing on the crypto worker. + pub(crate) fn derive_page_key( + &self, + envelope: &PageEnvelope, + ) -> Result { + let cache = &envelope.page.version.object.cache; + let key = self + .keys + .lease(Some(cache), envelope.key_id, KeyPurpose::Page)?; + key.derive_page_subkey(cache, envelope.key_id, &envelope.nonce.0) + .map_err(|_| Error::MissingKey) + } /// Authenticate the entire page before constructing a publishable lease. /// Futures stay on I/O, even though inputs/completions are Send: /// ```compile_fail diff --git a/cmd/racer-dataplane/src/store.rs b/cmd/racer-dataplane/src/store.rs index 7b2bacfcd0..36c875cfa6 100644 --- a/cmd/racer-dataplane/src/store.rs +++ b/cmd/racer-dataplane/src/store.rs @@ -101,6 +101,20 @@ pub struct ReadToken { page: PageId, location: RecordLocation, } +#[cfg(test)] +impl ReadToken { + /// A token for a mapping that need not exist, for tests that only route it. + pub(crate) fn for_test(page: PageId) -> Self { + Self { + page, + location: RecordLocation { + segment: page_alloc::SegmentId(0), + generation: page_alloc::Generation(1), + extent: page_alloc::Extent::new(0, 512).unwrap(), + }, + } + } +} impl StoreReader { pub fn metadata( &self, @@ -293,6 +307,141 @@ impl StoreReader { .map(|(copy, _)| copy)) }) } + + /// Locate a published v2 record's ciphertext on its device for a direct + /// (cuFile) read, reading and validating only the header through host + /// memory. Returns None on a miss, for v1 records, and for any record + /// this read path would also treat as a miss. The ciphertext checksum is + /// not verified here: the consumer must check + /// [`DirectRecord::checksum`] before trusting the bytes. + pub(crate) fn direct_read<'a>( + &'a self, + page: &'a PageId, + scope: &'a RequestScope, + reserve: impl Fn(usize) -> Result> + 'a, + ) -> Operation<'a, Option> { + Box::pin(async move { + scope.check()?; + let Some(entry) = self.metrics.lookup( + crate::telemetry::LookupTier::DiskIndex, + self.index.lookup(page), + )? + else { + return Ok(None); + }; + let location = entry.location.clone(); + let token = ReadToken { + page: page.clone(), + location: location.clone(), + }; + if self + .segments + .validate(location.segment, location.generation, &location.extent) + .is_err() + { + self.invalidate(&token)?; + return Ok(None); + } + // One lease rides with the header read; the other keeps the + // generation alive until the caller's direct transfer completes. + let (Ok(read_lease), Ok(hold)) = ( + self.segments.lease(location.segment, location.generation), + self.segments.lease(location.segment, location.generation), + ) else { + self.invalidate(&token)?; + return Ok(None); + }; + let prefix = location.extent.length().min(MAX_HEADER_BYTES); + let Ok(prefix_extent) = Extent::new(location.extent.offset(), prefix) else { + return Ok(None); + }; + let mut reserved = reserve(prefix); + if matches!(reserved, Err(Error::Overloaded)) { + self.slabs.reclaim_idle(); + reserved = reserve(prefix); + } + let reservation = reserved?; + let Ok(buffer) = self.slabs.allocate(prefix, reservation) else { + return Ok(None); + }; + let buffer = match self + .slabs + .read(&self.reactor, prefix_extent, buffer, read_lease, scope) + .await + { + Ok(buffer) => buffer, + Err( + Error::Io | Error::Os(_) | Error::CorruptRecord | Error::InvalidConfiguration, + ) => { + return Ok(None); + } + Err(error) => return Err(error), + }; + let decoded = match parse_prefix(buffer.bytes()?, location.extent) { + Ok(decoded) => decoded, + // A short prefix of a valid v1 record, another version, or + // damage: the host read path classifies and reports it. + Err(_) => return Ok(None), + }; + drop(buffer); + if decoded.header.envelope.page != *page + || decoded.header.generation != location.generation + || decoded.header.metadata != entry.metadata + || decoded.header.envelope.key_id != entry.key_id + { + self.corrupt_miss(); + self.invalidate(&token)?; + return Ok(None); + } + if self.index.lookup(page)?.is_none_or(|current| { + current.location != location || current.key_id != entry.key_id + }) { + return Ok(None); + } + let Some(ciphertext) = direct_ciphertext(&self.slabs, &location, &hold, &decoded)? + else { + return Ok(None); + }; + // cuFile reads whole 4 KiB blocks. v2 extents are 4 KiB multiples + // and the ciphertext ends the record, so rounding up stays inside it. + let rounded = ciphertext.length.next_multiple_of(CIPHERTEXT_ALIGNMENT); + let read_length = if decoded.ciphertext.start + rounded <= location.extent.length() { + rounded + } else { + ciphertext.length + }; + self.clock.mark_read(location.segment)?; + self.index + .retention() + .disk_read(page, u64::from(entry.metadata.page_length(page)?)); + Ok(Some(DirectRecord { + token, + metadata: entry.metadata.for_pin(), + envelope: decoded.header.envelope, + checksum: decoded.checksum, + ciphertext, + read_length, + lease: hold, + })) + }) + } +} + +/// A published v2 record located for a direct read by [`StoreReader::direct_read`]. +pub(crate) struct DirectRecord { + /// Invalidates exactly this mapping if the bytes fail verification. + pub token: ReadToken, + pub metadata: crate::model::ObjectMetadata, + pub envelope: PageEnvelope, + /// The stored CRC-64 of the sealed page (ciphertext and tag). + pub checksum: u64, + pub ciphertext: DirectCiphertext, + /// Bytes to transfer from `ciphertext.offset`: the ciphertext length + /// rounded up to whole 4 KiB blocks when the record holds them. + pub read_length: usize, + /// Keeps the segment generation, and so the bytes, from being recycled. + /// Hold it until the direct transfer completes. + pub lease: SegmentLease, } struct Dirty { @@ -1351,6 +1500,10 @@ pub const MAX_RAW_HEADER_BYTES: usize = 16384; pub const MAX_HEADER_BYTES: usize = MAX_RAW_HEADER_BYTES + 4096; /// Smallest record unit; every stored `header_len` is a multiple of it. const MIN_RECORD_UNIT: usize = 512; +/// Ciphertext alignment that cuFile needs for direct-to-GPU reads. A record +/// whose ciphertext starts at a multiple of this on the device can be read +/// straight into GPU memory; others are read through host memory. +pub const CIPHERTEXT_ALIGNMENT: usize = 4096; /// Ciphertext body that can be written in place, without a staging copy. /// @@ -1428,6 +1581,67 @@ pub struct DecodedRecord { pub ciphertext: std::ops::Range, pub checksum: u64, } +impl DecodedRecord { + /// Ciphertext byte range relative to the record start, if the record's + /// header block places the ciphertext at a CIPHERTEXT_ALIGNMENT boundary + /// and so supports direct-to-GPU reads. + pub fn direct_range(&self) -> Option> { + (self.header.format_version == FORMAT_VERSION + && self.ciphertext.start.is_multiple_of(CIPHERTEXT_ALIGNMENT)) + .then_some(self.ciphertext.start as u64..self.ciphertext.end as u64) + } +} +/// Where a published record's ciphertext lives on its backing file or device. +/// Returned only when the header block is a multiple of CIPHERTEXT_ALIGNMENT. +/// The descriptor is the worker-local O_DIRECT +/// descriptor the slab reads through, suitable for registration with cuFile. +pub struct DirectCiphertext { + pub descriptor: Rc, + /// Absolute byte offset of the first ciphertext byte on `descriptor`. + pub offset: u64, + /// Ciphertext length in bytes, including the AEAD tag. + pub length: usize, + /// Whether `offset` is a multiple of CIPHERTEXT_ALIGNMENT. This holds for + /// every such record whose physical start is 4 KiB aligned. See + /// `direct_ciphertext` for when that is guaranteed. + pub aligned: bool, +} +/// Translate a parsed record into its absolute on-device ciphertext location. +/// +/// Returns Ok(None) when the record's header block is not a multiple of +/// CIPHERTEXT_ALIGNMENT; such records must be read through host memory. +/// `lease` must be the lease for `location` and must stay alive until any +/// direct transfer completes. Record extents and header blocks are multiples +/// of the record unit, so the ciphertext is 4 KiB aligned on disk whenever the +/// record unit is a multiple of 4 KiB and the segment's physical start is 4 KiB +/// aligned. `aligned` reports the actual result rather than assuming it. +pub fn direct_ciphertext( + slabs: &Slab>, + location: &catalog::RecordLocation, + lease: &SegmentLease, + record: &DecodedRecord, +) -> Result> { + if record.header.extent != location.extent + || lease.id() != location.segment + || lease.generation() != location.generation + { + return Err(Error::CorruptRecord); + } + let Some(range) = record.direct_range() else { + return Ok(None); + }; + let (descriptor, physical) = slabs.locate(location.extent, lease)?; + let offset = physical + .offset() + .checked_add(range.start) + .ok_or(Error::CorruptRecord)?; + Ok(Some(DirectCiphertext { + descriptor, + offset, + length: (range.end - range.start) as usize, + aligned: offset.is_multiple_of(CIPHERTEXT_ALIGNMENT as u64), + })) +} struct RecordLayout { /// Unpadded header length, including the digest. header_bytes: usize, @@ -1438,14 +1652,21 @@ struct RecordLayout { /// `header_block` plus the unpadded ciphertext length. logical_bytes: usize, } -/// Record unit: the extent granularity, so every in-record offset that is a -/// multiple of it is both offset- and length-aligned on disk. +/// Record unit: a multiple of the extent granularity, so every in-record +/// offset that is a multiple of it is both offset- and length-aligned on disk. +/// It is at least CIPHERTEXT_ALIGNMENT, so header blocks and record extents are +/// 4 KiB multiples and ciphertext stays eligible for direct-to-GPU reads on +/// devices with smaller sectors. fn record_unit(alignment: Alignment) -> Result { let unit = alignment.extent(0, 1)?.length(); if !unit.is_multiple_of(MIN_RECORD_UNIT) { return Err(Error::InvalidConfiguration); } - Ok(unit) + if CIPHERTEXT_ALIGNMENT.is_multiple_of(unit) { + Ok(CIPHERTEXT_ALIGNMENT) + } else { + Ok(unit) + } } fn align_up(len: usize, unit: usize) -> Result { len.checked_next_multiple_of(unit) @@ -1456,7 +1677,13 @@ fn layout( generation: Generation, alignment: Alignment, ) -> Result { - let unit = record_unit(alignment)?; + layout_with_unit(page, generation, record_unit(alignment)?) +} +fn layout_with_unit( + page: &CiphertextCopy, + generation: Generation, + unit: usize, +) -> Result { let raw = raw_layout(page, generation)?; let header_block = align_up(raw, unit)?; if header_block > MAX_HEADER_BYTES { @@ -1608,11 +1835,34 @@ pub fn encode_at( generation: Generation, alignment: Alignment, offset: u64, - mut buffer: AlignedBuffer>, + buffer: AlignedBuffer>, ) -> Result { let layout = layout(page, generation, alignment)?; + encode_layout(page, generation, alignment, offset, buffer, layout) +} +/// Encode with the extent unit as the record unit, the layout written before +/// the record unit had a 4 KiB floor. Readers must keep accepting it. +#[cfg(test)] +pub(crate) fn encode_unit_aligned_at( + page: &CiphertextCopy, + generation: Generation, + alignment: Alignment, + offset: u64, + buffer: AlignedBuffer>, +) -> Result { + let layout = layout_with_unit(page, generation, alignment.extent(0, 1)?.length())?; + encode_layout(page, generation, alignment, offset, buffer, layout) +} +fn encode_layout( + page: &CiphertextCopy, + generation: Generation, + alignment: Alignment, + offset: u64, + mut buffer: AlignedBuffer>, + layout: RecordLayout, +) -> Result { let logical_bytes = layout.logical_bytes; - let extent = alignment.extent(offset, logical_bytes)?; + let extent = alignment.extent(offset, layout.header_block + layout.body_bytes)?; if extent.length() != layout.header_block + layout.body_bytes { return Err(Error::InvalidConfiguration); } @@ -1639,10 +1889,26 @@ pub fn parse( ) -> Result { parse_bytes(buffer.bytes()?, extent) } +/// Parse and validate a whole record of `extent`'s length. pub fn parse_bytes(bytes: &[u8], extent: Extent) -> Result { if bytes.len() != extent.length() { return Err(Error::CorruptRecord); } + parse_header(bytes, extent) +} +/// Parse and validate a record header from a prefix of the record at `extent`, +/// without touching the ciphertext. The prefix must cover the whole header +/// block; reading min(MAX_HEADER_BYTES, extent length) bytes always does. This +/// lets a direct-to-GPU reader validate framing through host memory and then +/// transfer only `DecodedRecord::ciphertext`. The ciphertext checksum is not +/// verified here. +pub fn parse_prefix(prefix: &[u8], extent: Extent) -> Result { + if prefix.len() > extent.length() { + return Err(Error::CorruptRecord); + } + parse_header(prefix, extent) +} +fn parse_header(bytes: &[u8], extent: Extent) -> Result { let mut r = Decoder(bytes); if r.take(8)? != MAGIC { return Err(Error::CorruptRecord); @@ -1725,7 +1991,7 @@ pub fn parse_bytes(bytes: &[u8], extent: Extent) -> Result { let logical_bytes = header_len .checked_add(ciphertext_length as usize) .ok_or(Error::CorruptRecord)?; - if logical_bytes > bytes.len() { + if logical_bytes > extent.length() { return Err(Error::CorruptRecord); } Ok(DecodedRecord { diff --git a/cmd/racer-dataplane/src/store/tests.rs b/cmd/racer-dataplane/src/store/tests.rs index 849d6fb6b7..3eea504114 100644 --- a/cmd/racer-dataplane/src/store/tests.rs +++ b/cmd/racer-dataplane/src/store/tests.rs @@ -3515,7 +3515,7 @@ mod records { generation, alignment, 1, - buffer(&admission, alignment, 1024), + buffer(&admission, alignment, 8192), ) .err(); assert_eq!(actual, Some(Error::CorruptRecord)); @@ -3540,7 +3540,7 @@ mod records { &page, Generation(7), alignment, - buffer(&admission, alignment, 1024), + buffer(&admission, alignment, 8192), ) .unwrap(); assert_eq!( @@ -3562,7 +3562,7 @@ mod records { Generation(0), alignment, 1, - buffer(&admission, alignment, 1024) + buffer(&admission, alignment, 8192) ) .err(), Some(Error::CorruptRecord) @@ -3693,10 +3693,11 @@ mod records { let admission = admission(); let page = page(&admission, 3, 0, "cache", "v1"); let alignment = Alignment::new(512, 512, 512).unwrap(); - let encoded = encode( + let encoded = crate::store::encode_unit_aligned_at( &page, Generation(7), alignment, + 0, buffer(&admission, alignment, 1024), ) .unwrap(); @@ -3722,7 +3723,7 @@ mod records { &page, Generation(7), alignment, - buffer(&admission, alignment, 1024), + buffer(&admission, alignment, 8192), ) .unwrap(); let parsed = parse(&encoded.buffer, encoded.header.extent).unwrap(); @@ -3751,7 +3752,7 @@ mod records { &page, Generation(7), alignment, - buffer(&admission, alignment, 1024), + buffer(&admission, alignment, 8192), ) .unwrap(); let original = encoded.buffer.bytes().unwrap(); @@ -3799,19 +3800,20 @@ mod records { )); } - /// Encode `page(3, 0, "cache", etag)` at 512-byte alignment. + /// Encode `page(3, 0, "cache", etag)` with a 512-byte record unit, the + /// densest layout readers accept. fn encoded_v2( admission: &flow_control::Quotas, etag: &str, ) -> (Vec, Extent) { let page = page(admission, 3, 0, "cache", etag); let alignment = Alignment::new(512, 512, 512).unwrap(); - let length = record_length(&page, alignment).unwrap(); - let encoded = encode( + let encoded = crate::store::encode_unit_aligned_at( &page, Generation(7), alignment, - buffer(admission, alignment, length), + 0, + buffer(admission, alignment, 1024), ) .unwrap(); ( @@ -3833,11 +3835,13 @@ mod records { page.metadata.content_type = content_type .map(|v| crate::model::ContentType::parse(v.as_bytes()).unwrap()); let extra = content_type.map_or(0, str::len); + // The record unit has a 4 KiB floor for direct reads. + let record_unit = unit.max(crate::store::CIPHERTEXT_ALIGNMENT); let block = header_block_len(&page, alignment).unwrap(); - assert_eq!(block, (real + extra).next_multiple_of(unit)); - assert_eq!(body_len(&page, alignment).unwrap(), unit); + assert_eq!(block, (real + extra).next_multiple_of(record_unit)); + assert_eq!(body_len(&page, alignment).unwrap(), record_unit); let record = record_length(&page, alignment).unwrap(); - assert_eq!(record, block + unit); + assert_eq!(record, block + record_unit); let mut staging = buffer(&admission, alignment, record); staging.bytes_mut().unwrap().fill(0xa5); let encoded = @@ -3942,6 +3946,43 @@ mod records { admission.reclaim_buffers(); } + #[test] + fn prefix_parse_and_direct_range_cover_new_and_unit_aligned_records() { + let admission = admission(); + for unit in [512, 4096] { + let alignment = Alignment::new(unit, unit as u64, unit).unwrap(); + let page = page(&admission, 3, 0, "cache", "v1"); + let length = record_length(&page, alignment).unwrap(); + let encoded = encode( + &page, + Generation(7), + alignment, + buffer(&admission, alignment, length), + ) + .unwrap(); + let bytes = encoded.buffer.bytes().unwrap(); + let extent = encoded.header.extent; + let full = parse_bytes(bytes, extent).unwrap(); + let block = header_block_len(&page, alignment).unwrap(); + let prefix = parse_prefix(&bytes[..block], extent).unwrap(); + assert_eq!(prefix.ciphertext, full.ciphertext); + assert_eq!(prefix.header.metadata, full.header.metadata); + assert!(parse_prefix(&bytes[..block - 1], extent).is_err()); + let mut long = bytes.to_vec(); + long.push(0); + assert!(parse_prefix(&long, extent).is_err()); + // The 4 KiB record unit floor makes every new record eligible. + assert_eq!(full.direct_range(), Some(4096..4096 + 19)); + } + // Records written with a 512-byte header block still parse, but must + // be read through host memory. + let (bytes, extent) = encoded_v2(&admission, "v1"); + let old = parse_bytes(&bytes, extent).unwrap(); + assert_eq!(old.ciphertext, 512..512 + 19); + assert_eq!(old.direct_range(), None); + admission.reclaim_buffers(); + } + #[test] fn malformed_frames_are_rejected_without_unbounded_allocations() { for len in [1, 16, 512] { diff --git a/cmd/racer-dataplane/src/telemetry.rs b/cmd/racer-dataplane/src/telemetry.rs index 4852561b62..e589c5bb5d 100644 --- a/cmd/racer-dataplane/src/telemetry.rs +++ b/cmd/racer-dataplane/src/telemetry.rs @@ -3310,6 +3310,7 @@ metrics! { Event, EVENTS, EVENT_COUNT; Self::RdmaSentBytes => "racer_rdma_sent_bytes_total", Self::RdmaPagesReceived => "racer_rdma_pages_received_total", Self::RdmaReceivedBytes => "racer_rdma_received_bytes_total", + Self::RdmaPagesLanded => "racer_rdma_pages_landed_total", Self::RdmaHttpFallback => "racer_rdma_http_fallback_total", Self::RdmaGrantCount => "racer_rdma_grant_nanoseconds_count", Self::RdmaGrantNs => "racer_rdma_grant_nanoseconds_sum", @@ -3348,6 +3349,8 @@ metrics! { Event, EVENTS, EVENT_COUNT; Self::FillChecksumCount => "racer_fill_checksum_nanoseconds_count", Self::FillChecksumNs => "racer_fill_checksum_nanoseconds_sum", Self::DiagnosticHardware => "racer_diagnostic_hardware_total", + Self::GpuDirectDiskRead => "racer_gpu_direct_disk_reads_total", + Self::GpuDirectDiskReadBytes => "racer_gpu_direct_disk_read_bytes_total", } metrics! { Gauge, GAUGES, GAUGE_COUNT; Self::WarmTrackedJobs => "racer_warm_tracked_jobs", diff --git a/cmd/racer-dataplane/src/test_support.rs b/cmd/racer-dataplane/src/test_support.rs index 74c3dbd720..c0eefcbfe3 100644 --- a/cmd/racer-dataplane/src/test_support.rs +++ b/cmd/racer-dataplane/src/test_support.rs @@ -310,6 +310,9 @@ pub mod cluster { memory_pressure: true, pprof_enabled: false, enable_rdma, + enable_gpu: crate::config::GpuMode::Disabled, + gpu_staging_bytes: crate::config::DEFAULT_GPU_STAGING_BYTES, + gpu_landing_bytes: crate::config::DEFAULT_GPU_LANDING_BYTES, control_endpoint: "https://control.invalid".into(), peer_listen: "127.0.0.1:0".parse().unwrap(), diagnostics_listen: "127.0.0.1:0".parse().unwrap(), @@ -489,6 +492,12 @@ pub(crate) struct ReadWorker { writer: Rc, memory: Rc, cache: racer_control_wire::CacheId, + admission: Rc>, + clock: Rc, + reactor: Rc, + metrics: crate::telemetry::Metrics, + /// Keep cached pages and queued disk writes between polls. + retain: std::cell::Cell, } /// Publish a topology-only fixture through the real generic publication authority. @@ -553,7 +562,7 @@ impl ReadWorker { let segments = Rc::new(page_alloc::Segments::new(64 * 1024 * 1024)); // Reads use an empty disk index. No slabs need to be opened or written. let slabs = Rc::new(page_alloc::Slab::new( - origin.root.join("slabs/worker-0-slab-0.dat"), + Self::disk_root(&origin).join("slabs/worker-0-slab-0.dat"), 256 * 1024 * 1024, 64 * 1024 * 1024, crate::model::PAGE_BYTES as usize + crate::store::MAX_HEADER_BYTES + 16, @@ -566,8 +575,9 @@ impl ReadWorker { reactor.clone(), availability.clone(), )); + let clock = Rc::new(SegmentClock::new(index.clone(), segments.clone(), 1)); let disk = Rc::new(StoreReader::new( - Rc::new(SegmentClock::new(index.clone(), segments.clone(), 1)), + clock.clone(), index.clone(), segments, slabs, @@ -578,6 +588,10 @@ impl ReadWorker { let (port, engine) = crate::security::pair(WorkerId(0), 0, std::num::NonZeroUsize::new(16).unwrap()); let crypto = Rc::new(CryptoClient::new(port)); + let metrics = crate::telemetry::Metrics::default(); + crypto.set_metrics(metrics.clone()); + let worker_admission = admission.clone(); + let worker_reactor = reactor.clone(); let credentials = Rc::new(CredentialCrypto::new(keys.clone(), admission.clone())); let candidates = Rc::new(CandidatePolicy::new( keys.node().clone(), @@ -593,19 +607,22 @@ impl ReadWorker { buffers.clone(), ); let memory = Rc::new(MemoryCache::new(buffers.clone(), availability.clone())); - let fill = Rc::new(Fill::new(FillDependencies { - memory: memory.clone(), - buffers, - disk, - writer: writer.clone(), - origin: client.clone(), - candidates: candidates.clone(), - flights: Rc::new(Flights::new(admission.clone(), availability.clone())), - crypto: Rc::new(PageCrypto::new(keys, crypto.clone())), - credentials: credentials.clone(), - admission, - metadata_owner: directory.clone(), - })); + let fill = Rc::new( + Fill::new(FillDependencies { + memory: memory.clone(), + buffers, + disk, + writer: writer.clone(), + origin: client.clone(), + candidates: candidates.clone(), + flights: Rc::new(Flights::new(admission.clone(), availability.clone())), + crypto: Rc::new(PageCrypto::new(keys, crypto.clone())), + credentials: credentials.clone(), + admission, + metadata_owner: directory.clone(), + }) + .with_metrics(metrics.clone()), + ); let metadata = Rc::new(MetadataService::new( candidates, client, @@ -640,9 +657,103 @@ impl ReadWorker { writer, memory, cache: cache.id, + admission: worker_admission, + clock, + reactor: worker_reactor, + metrics, + retain: std::cell::Cell::new(false), } } + /// Keep cached pages and queued disk writes between polls, so a test can + /// arrange memory and disk contents across requests. + pub fn retain(&self, retain: bool) { + self.retain.set(retain); + } + + /// This worker's page cache. + pub fn memory(&self) -> &Rc { + &self.memory + } + + /// CPU page decryptions started on this worker's crypto engine. + pub fn cpu_decrypts(&self) -> u64 { + self.metrics + .count(crate::telemetry::Event::CryptoDecryptStarted) + } + + /// This worker's count of `event`. + pub fn metric(&self, event: crate::telemetry::Event) -> u64 { + self.metrics.count(event) + } + + /// Open this worker's slab so pages can be written to and read from disk. + pub fn open_disk(&self) { + std::fs::create_dir_all(Self::disk_root(&self.origin).join("slabs")).unwrap(); + self.writer + .configure(self.admission.clone(), self.clock.clone(), 8, 64) + .unwrap(); + let _ = futures::executor::block_on(self.writer.open()).unwrap(); + } + + /// Write `copy` to disk and wait until it is published. + pub fn persist(&self, copy: crate::memory::CiphertextCopy) { + use crate::admission::ResourceClass; + let id = copy.ciphertext.envelope().page.clone(); + let dirty = self + .admission + .reserve( + Some(&id.version.object.cache), + ResourceClass::DirtyCiphertext, + copy.ciphertext.bytes().len(), + ) + .unwrap(); + self.writer.enqueue(copy, dirty).unwrap(); + let scope = crate::runtime::RequestScope::new( + crate::model::RequestId([1; 16]), + std::time::Instant::now() + std::time::Duration::from_secs(10), + ) + .unwrap(); + let mut progress = self.writer.progress(1, &scope); + let waker = futures::task::noop_waker(); + let mut cx = Context::from_waker(&waker); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while self.writer.index().lookup(&id).unwrap().is_none() { + if progress.as_mut().poll(&mut cx).is_ready() { + progress = self.writer.progress(1, &scope); + } + self.reactor.poll_budgeted(32).unwrap(); + assert!( + std::time::Instant::now() < deadline, + "disk write did not finish" + ); + std::thread::yield_now(); + } + } + + /// The published disk location of a page, if any. + pub fn disk_location( + &self, + page: &crate::model::PageId, + ) -> Option { + self.writer + .index() + .lookup(page) + .unwrap() + .map(|entry| entry.location) + } + + /// The slab file backing this worker's disk. + pub fn slab_path(&self) -> std::path::PathBuf { + Self::disk_root(&self.origin).join("slabs/worker-0-slab-0.dat") + } + + /// The origin's directory by its real path. The slab refuses to open + /// through the `/proc` symlink that `root` is. + fn disk_root(origin: &AdapterOrigin) -> std::path::PathBuf { + std::fs::read_link(&origin.root).unwrap_or_else(|_| origin.root.clone()) + } + /// Whether every command submitted to this worker has completed. pub fn is_drained(&self) -> bool { self.endpoint.borrow().is_drained() && self.drivers.pending() == 0 @@ -659,6 +770,9 @@ impl ReadWorker { ) .unwrap(); self.crypto.poll_budgeted(64).unwrap(); + if self.retain.get() { + return; + } // Exercise acquisition/authentication, not persistence or cache retention. // Delivered leases remain charged even after the cache drops its copy. self.writer.discard_unsubmitted(); diff --git a/cmd/racer-dataplane/verbs/native/verbs.c b/cmd/racer-dataplane/verbs/native/verbs.c index edb87e16c4..1639eb02d9 100644 --- a/cmd/racer-dataplane/verbs/native/verbs.c +++ b/cmd/racer-dataplane/verbs/native/verbs.c @@ -64,10 +64,14 @@ _Static_assert(IBV_WC_SEND == 0 && IBV_WC_RDMA_WRITE == 1 && IBV_WC_BIND_MW == 5 /* Private handles. Rust only holds pointers to these. */ struct rdma_verbs_device { struct ibv_context *ctx; struct ibv_pd *pd; }; struct rdma_verbs_qp { struct ibv_qp *qp; struct ibv_cq *cq; }; -struct rdma_verbs_mr { struct ibv_mr *mr; void *bytes; size_t length; }; +/* `bytes` is NULL for an implicit or dma-buf region. `addr` is the address + * windows bind from: the buffer pointer, or the dma-buf iova. `device` is set + * for a dma-buf region, which only receives through windows. */ +struct rdma_verbs_mr { struct ibv_mr *mr; void *bytes; size_t length; uint64_t addr; int device; }; -/* ABI version. Rust refuses to load any other value. */ -uint32_t rdma_verbs_abi(void) { return 4; } +/* ABI version. Rust refuses to load any other value. + * Version 5 added rdma_verbs_register_dmabuf. */ +uint32_t rdma_verbs_abi(void) { return 5; } /* Distinct from negative errno: cleanup leaked a resource. Rust must keep * the admission owner and adapter loaded. Stop before another resource leaks. */ @@ -246,11 +250,30 @@ void *rdma_verbs_register(struct rdma_verbs_device *d, uint64_t length, uint32_t if (!m) return NULL; if (posix_memalign(&m->bytes, 4096, (size_t)length)) { free(m); return NULL; } memset(m->bytes, 0, (size_t)length); m->length = (size_t)length; + m->addr = (uintptr_t)m->bytes; m->mr = ibv_reg_mr(d->pd, m->bytes, (size_t)length, flags); if (!m->mr) { free(m->bytes); free(m); return NULL; } return m; } -/* Deregister and free the buffer. */ +/* Register `length` bytes of a dma-buf, starting at `offset` in it, as page + * buffers (LOCAL_WRITE | MW_BIND, so peers write only through windows). + * Windows over it bind from `iova`. The caller keeps the memory behind the + * dma-buf alive until deregister succeeds. The kernel holds its own reference + * to the dma-buf, so `fd` may be closed after this returns. The CPU cannot + * reach the memory: rdma_verbs_bytes returns NULL, and send, write, and + * receive posts reject the region. Needs rdma-core 34 or later. */ +void *rdma_verbs_register_dmabuf(struct rdma_verbs_device *d, uint64_t offset, uint64_t length, + uint64_t iova, int fd) { + if (!length || length > SIZE_MAX || fd < 0) return NULL; + struct rdma_verbs_mr *m = calloc(1, sizeof(*m)); + if (!m) return NULL; + m->length = (size_t)length; m->addr = iova; m->device = 1; + m->mr = ibv_reg_dmabuf_mr(d->pd, offset, (size_t)length, iova, fd, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_MW_BIND); + if (!m->mr) { free(m); return NULL; } + return m; +} +/* Deregister and free the buffer, if this side allocated one. */ int rdma_verbs_deregister(struct rdma_verbs_mr *m) { int rc = ibv_dereg_mr(m->mr); if (!rc) { free(m->bytes); free(m); } @@ -279,7 +302,7 @@ void *rdma_verbs_register_implicit(struct rdma_verbs_device *d) { m->bytes = NULL; m->length = SIZE_MAX; return m; } -/* The buffer behind a registered region. */ +/* The buffer behind a registered region. NULL for an implicit or dma-buf region. */ void *rdma_verbs_bytes(struct rdma_verbs_mr *m) { return m->bytes; } /* Allocate a type 2 memory window. Writes the key to use for its next bind. */ void *rdma_verbs_window(struct rdma_verbs_device *d, uint32_t *key) { @@ -289,6 +312,8 @@ void *rdma_verbs_window(struct rdma_verbs_device *d, uint32_t *key) { } /* Free a memory window. */ int rdma_verbs_window_free(struct ibv_mw *mw) { return ibv_dealloc_mw(mw); } +/* True if `m` has CPU-reachable or implicit memory a post may use as an SGE. */ +static int host_sge(const struct rdma_verbs_mr *m) { return !m->device; } /* True if [offset, offset + length) is a nonempty range inside `m`. */ static int in_region(const struct rdma_verbs_mr *m, uint64_t offset, uint32_t length) { return length && offset <= m->length && length <= m->length - offset; @@ -306,7 +331,7 @@ int rdma_verbs_bind(struct rdma_verbs_qp *q, struct ibv_mw *mw, struct rdma_verb wr.send_flags = IBV_SEND_SIGNALED | IBV_SEND_FENCE; wr.bind_mw.mw = mw; wr.bind_mw.rkey = key; wr.bind_mw.bind_info.mr = m->mr; - wr.bind_mw.bind_info.addr = (uintptr_t)m->bytes + offset; + wr.bind_mw.bind_info.addr = m->addr + offset; wr.bind_mw.bind_info.length = length; wr.bind_mw.bind_info.mw_access_flags = IBV_ACCESS_REMOTE_WRITE; return ibv_post_send(q->qp, &wr, &bad); @@ -325,7 +350,7 @@ int rdma_verbs_invalidate(struct rdma_verbs_qp *q, uint32_t key, uint64_t id) { int rdma_verbs_write_imm(struct rdma_verbs_qp *q, struct rdma_verbs_mr *m, uint64_t offset, uint32_t length, uint64_t address, uint32_t key, uint32_t imm, uint64_t id) { - if (!in_region(m, offset, length)) return EINVAL; + if (!host_sge(m) || !in_region(m, offset, length)) return EINVAL; struct ibv_sge sge = { .addr = (uintptr_t)m->bytes + offset, .length = length, .lkey = m->mr->lkey }; struct ibv_send_wr wr = {0}, *bad = NULL; @@ -338,7 +363,7 @@ int rdma_verbs_write_imm(struct rdma_verbs_qp *q, struct rdma_verbs_mr *m, uint6 * the peer's posted receives. */ int rdma_verbs_send(struct rdma_verbs_qp *q, struct rdma_verbs_mr *m, uint64_t offset, uint32_t length, uint64_t id) { - if (!in_region(m, offset, length)) return EINVAL; + if (!host_sge(m) || !in_region(m, offset, length)) return EINVAL; struct ibv_sge sge = { .addr = (uintptr_t)m->bytes + offset, .length = length, .lkey = m->mr->lkey }; struct ibv_send_wr wr = {0}, *bad = NULL; @@ -349,7 +374,7 @@ int rdma_verbs_send(struct rdma_verbs_qp *q, struct rdma_verbs_mr *m, uint64_t o /* Post a receive into `length` bytes of `m` at `offset`. */ int rdma_verbs_post_recv(struct rdma_verbs_qp *q, struct rdma_verbs_mr *m, uint64_t offset, uint32_t length, uint64_t id) { - if (!in_region(m, offset, length)) return EINVAL; + if (!host_sge(m) || !in_region(m, offset, length)) return EINVAL; struct ibv_sge sge = { .addr = (uintptr_t)m->bytes + offset, .length = length, .lkey = m->mr->lkey }; struct ibv_recv_wr wr = {0}, *bad = NULL; diff --git a/cmd/racer-dataplane/verbs/src/ffi.rs b/cmd/racer-dataplane/verbs/src/ffi.rs index 4927f9ac9c..018cf96f77 100644 --- a/cmd/racer-dataplane/verbs/src/ffi.rs +++ b/cmd/racer-dataplane/verbs/src/ffi.rs @@ -3,7 +3,7 @@ //! Only the native thread uses these. Each type owns one RDMA object and frees //! it on drop. If freeing fails, the object is leaked on purpose, along with //! everything it depends on, because the NIC may still be using it. -use crate::{Error, GuardOwner, Result}; +use crate::{DeviceMemory, Error, GuardOwner, Result}; use std::{ cell::{Cell, RefCell}, @@ -109,8 +109,8 @@ struct Completion { const WC_IMM: u32 = 1; /// Native adapter ABI version this crate speaks. -#[cfg(all(feature = "native", target_os = "linux"))] -const ABI_VERSION: u32 = 4; +#[cfg(any(test, all(feature = "native", target_os = "linux")))] +const ABI_VERSION: u32 = 5; /// Completion opcodes (`enum ibv_wc_opcode`), pinned by C static asserts. pub(crate) mod opcode { @@ -166,6 +166,10 @@ impl Access { /// Never passed to `rdma_verbs_register`; only a local write source. pub(crate) const IMPLICIT: Self = Self(8); + /// Marks a dma-buf page region from `rdma_verbs_register_dmabuf`. Never + /// passed to `rdma_verbs_register`; peers write it through windows only. + pub(crate) const DEVICE: Self = Self(1 | 16); + /// The raw ABI value. pub(crate) fn bits(self) -> u32 { self.0 @@ -257,6 +261,8 @@ native_api! { register_implicit: unsafe extern "C" fn(*mut c_void) -> *mut c_void, + register_dmabuf: unsafe extern "C" fn(*mut c_void, u64, u64, u64, c_int) -> *mut c_void, + deregister: unsafe extern "C" fn(*mut c_void) -> c_int, bytes: unsafe extern "C" fn(*mut c_void) -> *mut u8, @@ -489,11 +495,18 @@ fn discover_with_api( /// /// Posts do not track which ranges the NIC may touch; their caller must. /// Holds the node's guard and leaks it if deregister fails. +/// +/// A dma-buf region (`memory` is set) has no CPU mapping. It can only be +/// bound to windows for receives; sends, writes, and receives reject it. pub(crate) struct NativeRegion { device: Rc, raw: NonNull, + /// The dma-buf owner. Kept alive while registered, and leaked with the + /// region if deregister fails. + memory: Option, + /// Allocated size. length: usize, @@ -520,12 +533,56 @@ impl NativeRegion { Ok(Rc::new(Self { device, raw, + memory: None, length, quota: Some(quota), access, })) } + /// Register all of `memory`'s dma-buf range as page buffers. `memory` + /// stays alive until deregister succeeds, or forever if it fails. + pub(crate) fn dmabuf( + device: Rc, + memory: DeviceMemory, + quota: Arc, + ) -> Result> { + use std::os::fd::AsRawFd; + + let length = memory.length(); + if length == 0 || u64::try_from(length).is_err() { + return Err(Error::InvalidRange); + } + let raw = NonNull::new(unsafe { + (device.api.register_dmabuf)( + device.raw.as_ptr(), + memory.offset(), + length as u64, + memory.iova(), + memory.fd().as_raw_fd(), + ) + }) + .ok_or(Error::Unavailable)?; + Ok(Rc::new(Self { + device, + raw, + memory: Some(memory), + length, + quota: Some(quota), + access: Access::DEVICE, + })) + } + + /// The dma-buf owner, if this is a dma-buf region. + pub(crate) fn memory(&self) -> Option<&DeviceMemory> { + self.memory.as_ref() + } + + /// True for a dma-buf region from [`Self::dmabuf`]. + pub(crate) fn is_device(&self) -> bool { + self.memory.is_some() + } + /// Register the whole process address space as one implicit on-demand /// paging MR, for zero-copy write sources. `None` if the device lacks /// implicit ODP for RC writes. @@ -538,6 +595,7 @@ impl NativeRegion { Some(Rc::new(Self { device, raw, + memory: None, length: usize::MAX, quota: Some(quota), access: Access::IMPLICIT, @@ -552,7 +610,11 @@ impl NativeRegion { /// The start of the registered buffer, valid while `self` lives. /// /// The caller must not read or write any range the NIC may still access. + /// Null for a dma-buf region. pub(crate) fn as_ptr(&self) -> *mut u8 { + if self.is_device() { + return std::ptr::null_mut(); + } unsafe { (self.device.api.bytes)(self.raw.as_ptr()) } } @@ -566,6 +628,9 @@ impl NativeRegion { /// The address peers use to write here. pub(crate) fn address(&self) -> u64 { + if let Some(memory) = &self.memory { + return memory.iova(); + } #[cfg(any(test, feature = "simulation"))] if simulation::is_api(&self.device.api) { return simulation::address(self.raw); @@ -580,10 +645,11 @@ impl NativeRegion { } impl Drop for NativeRegion { - /// Deregister. On failure, leak the device and guard. + /// Deregister. On failure, leak the device, guard, and dma-buf owner. fn drop(&mut self) { if unsafe { (self.device.api.deregister)(self.raw.as_ptr()) } != 0 { std::mem::forget(self.device.clone()); + std::mem::forget(self.memory.take()); self.quota.as_ref().unwrap().quarantine(); std::mem::forget(self.quota.take()); } @@ -945,6 +1011,7 @@ impl NativeQueuePair { post: impl FnOnce() -> c_int, ) -> Result<()> { if (region.is_implicit() && (recv || opcode != opcode::RDMA_WRITE)) + || (region.is_device() && !matches!(opcode, opcode::BIND_MW | opcode::LOCAL_INV)) || !Rc::ptr_eq(®ion.device, &self.device) || !region.contains(offset, length) || self.pending.borrow().contains_key(&id) @@ -1195,6 +1262,20 @@ mod tests { assert!(api.library.is_some(), "must load C adapter, not simulation"); } + #[test] + /// The C adapter reports the ABI version Rust requires and exports the + /// dma-buf registration with the signature Rust calls. + fn c_adapter_abi_version_and_dmabuf_signature_match() { + let source = include_str!("../native/verbs.c"); + assert!(source.contains(&format!( + "uint32_t rdma_verbs_abi(void) {{ return {ABI_VERSION}; }}" + ))); + assert!(source.contains( + "void *rdma_verbs_register_dmabuf(struct rdma_verbs_device *d, uint64_t offset, \ + uint64_t length,\n uint64_t iova, int fd)" + )); + } + #[test] /// Reject queue pair numbers outside the native 24-bit field. fn endpoint_rejects_invalid_native_parameters() { @@ -1290,6 +1371,17 @@ pub(crate) mod lifetime_tests { std::ptr::null_mut() } + /// The fixture device has no dma-buf support. + unsafe extern "C" fn register_dmabuf( + _: *mut c_void, + _: u64, + _: u64, + _: u64, + _: c_int, + ) -> *mut c_void { + std::ptr::null_mut() + } + /// Allocate a fixture queue pair with a fixed valid number. unsafe extern "C" fn qp(_: *mut c_void, _: u8, _: u32, _: u32, qpn: *mut u32) -> *mut c_void { unsafe { @@ -1452,6 +1544,7 @@ pub(crate) mod lifetime_tests { qp_free, register, register_implicit, + register_dmabuf, deregister, bytes, window, diff --git a/cmd/racer-dataplane/verbs/src/lib.rs b/cmd/racer-dataplane/verbs/src/lib.rs index 4ccc6e4b91..a0e6ba12a6 100644 --- a/cmd/racer-dataplane/verbs/src/lib.rs +++ b/cmd/racer-dataplane/verbs/src/lib.rs @@ -77,7 +77,7 @@ pub use node::{ BACKOFF_MAX, BACKOFF_MIN, BACKOFF_RESET, COMMAND_QUEUE, Client, LinkKey, LinkSelector, LinkState, NodeConfig, Opened, Owner, OwnerPort, SETUP_TIMEOUT, Setup, node, }; -pub use pool::{BufferState, RegisteredPage}; +pub use pool::{BufferState, DevicePage, RegisteredPage}; #[cfg(any(test, feature = "simulation"))] pub use ffi::simulation; @@ -100,6 +100,37 @@ pub fn inventory() -> Result> { .collect()) } +/// Memory outside this crate that the NIC reaches through a Linux dma-buf, +/// such as GPU memory exported with `cuMemGetHandleForAddressRange`. +/// +/// A device pool built over it (see [`Client::attach_device`]) lets peers +/// write pages straight into this memory. The CPU never maps or copies it: +/// device pages can only receive, and their bytes are only reachable through +/// the owner (for example a GPU kernel). +/// +/// The crate keeps the owning [`DeviceMemory`] alive while the memory is +/// registered. If deregistration fails, the owner is leaked on purpose, like a +/// [`Guard`], so the memory is never freed while the NIC may still write it. +pub trait DmabufMemory: Send + Sync { + /// The dma-buf descriptor. Read only on the native thread, during + /// registration. The kernel keeps its own reference after that. + fn fd(&self) -> std::os::fd::BorrowedFd<'_>; + + /// Byte offset of this memory inside the dma-buf. + fn offset(&self) -> u64; + + /// Usable bytes from [`Self::offset`]. Split into page buffers of the + /// node's buffer length. + fn length(&self) -> usize; + + /// The address the NIC uses for the first byte. Windows over this memory + /// bind from it. + fn iova(&self) -> u64; +} + +/// The owner of a device pool's memory. See [`DmabufMemory`]. +pub type DeviceMemory = Arc; + /// Any value you want kept alive as long as a slot's RDMA resources live. /// /// Use it to count admission. If a resource is leaked, its guard leaks too. diff --git a/cmd/racer-dataplane/verbs/src/link.rs b/cmd/racer-dataplane/verbs/src/link.rs index c6dcb0bd9e..ccc7d8be85 100644 --- a/cmd/racer-dataplane/verbs/src/link.rs +++ b/cmd/racer-dataplane/verbs/src/link.rs @@ -22,7 +22,7 @@ use crate::{ ffi::{ Access, MemoryWindow, NativeDevice, NativeQueuePair, NativeRegion, WorkCompletion, opcode, }, - pool::{Buffer, BufferState, Pool, RegisteredPage}, + pool::{Buffer, BufferState, DevicePage, Pool, RegisteredPage}, }; /// A link not `Ready` this long after creation breaks. @@ -63,6 +63,10 @@ pub enum Arrival { /// CPU access. Page(RegisteredPage), + /// Like [`Self::Page`], for a grant made with [`crate::Client::grant_device`]: + /// the bytes are in device memory and the CPU never touched them. + Device(DevicePage), + /// The peer sent `Release`. The buffer is already back in the pool. Released, } @@ -383,6 +387,9 @@ pub(crate) struct Link { windows: Vec>, + /// The device, for windows over a device pool, made on first use. + device: Rc, + control: Rc, epoch: Epoch, @@ -414,6 +421,10 @@ pub(crate) struct Link { free_windows: Vec, + /// Free windows tied to the device pool. A window stays tied to the + /// first region it binds, so device grants never reuse host windows. + free_device_windows: Vec, + free_slots: Vec, grants: BTreeMap, @@ -491,6 +502,8 @@ impl Link { qp, free_windows: (0..windows.len()).rev().collect(), windows, + device: device.clone(), + free_device_windows: Vec::new(), control, epoch, role, @@ -603,10 +616,15 @@ impl Link { } /// Bind a window over a free buffer and queue a `Grant`. `done` gets the - /// tag and arrival op once the `Grant` is posted. + /// tag and arrival op once the `Grant` is posted. With `device`, the + /// buffer comes from the device pool and the arrival is + /// [`Arrival::Device`]: fails with [`Error::Unavailable`] if the rail has + /// no device pool and [`Error::Overloaded`] if none is free. + #[allow(clippy::too_many_arguments)] pub(crate) fn grant( &mut self, pool: &Pool, + device: bool, transfer: [u8; 16], length: u32, deadline: Instant, @@ -628,12 +646,42 @@ impl Link { return; } - let Some(buffer) = pool.take(BufferState::Binding) else { + if device && pool.device_shared().is_none() { + let _ = done.complete(Err(Error::Unavailable)); + return; + } + + let window = if device { + match self.free_device_windows.pop() { + Some(window) => window, + None => match MemoryWindow::new(&self.device) { + // At most one device window per concurrent grant, so + // device windows never exceed the grant cap. + Ok(window) => { + self.windows.push(window); + self.windows.len() - 1 + } + Err(error) => { + let _ = done.complete(Err(error)); + return; + } + }, + } + } else { + self.free_windows.pop().expect("a window per grant") + }; + + let taken = if device { + pool.take_device(BufferState::Binding) + } else { + pool.take(BufferState::Binding) + }; + let Some(buffer) = taken else { + self.free_window(device, window); let _ = done.complete(Err(Error::Overloaded)); return; }; - let window = self.free_windows.pop().expect("a window per grant"); let tag = self.allocate_tag(); self.last_used = now; self.grants.insert( @@ -653,6 +701,15 @@ impl Link { self.outbox.push_back(Outgoing::Bind(tag)); } + /// Return `window` to its free list. + fn free_window(&mut self, device: bool, window: usize) { + if device { + self.free_device_windows.push(window); + } else { + self.free_windows.push(window); + } + } + /// Next nonzero tag not in use. fn allocate_tag(&mut self) -> u32 { loop { @@ -880,22 +937,27 @@ impl Link { } InFlight::Invalidate(tag) => { let mut entry = self.grants.remove(&tag).ok_or(Break::Error)?; - self.free_windows.push(entry.window); let buffer = entry.buffer.take().ok_or(Break::Error)?; + self.free_window(buffer.is_device(), entry.window); let GrantState::Invalidating(outcome) = entry.state else { return Err(Break::Error); }; match outcome { Outcome::Written(len) => { - let page = pool.own(buffer, len as usize).map_err(|buffer| { + let page = if buffer.is_device() { + pool.own_device(buffer, len as usize).map(Arrival::Device) + } else { + pool.own(buffer, len as usize).map(Arrival::Page) + } + .map_err(|buffer| { self.quarantine.push(buffer); Break::Error })?; if let Some(arrival) = entry.arrival.take() { // A dropped op hands the page back; dropping it frees the buffer. - let _ = arrival.complete(Ok(Arrival::Page(page))); + let _ = arrival.complete(Ok(page)); } } Outcome::Released => { @@ -1172,7 +1234,7 @@ impl Link { let key = self.qp.bind_window( wr(BIND, tag), &self.windows[entry.window], - pool.region(), + pool.region_of(buffer), pool.offset(buffer), entry.length, )?; @@ -1229,7 +1291,7 @@ impl Link { let message = ControlKind::Grant { tag, rkey: entry.key, - addr: pool.region().address() + pool.offset(buffer), + addr: pool.region_of(buffer).address() + pool.offset(buffer), length: entry.length, transfer_id: entry.transfer, }; diff --git a/cmd/racer-dataplane/verbs/src/link/tests.rs b/cmd/racer-dataplane/verbs/src/link/tests.rs index 4c11253479..8bed8cb418 100644 --- a/cmd/racer-dataplane/verbs/src/link/tests.rs +++ b/cmd/racer-dataplane/verbs/src/link/tests.rs @@ -142,7 +142,7 @@ fn poll(op: &Op) -> Option> { fn grant(side: &mut Side, length: u32, deadline: Instant, now: Instant) -> Op { let (granted, done) = op(); side.link - .grant(&side.pool, [7; 16], length, deadline, done, now); + .grant(&side.pool, false, [7; 16], length, deadline, done, now); granted } @@ -182,7 +182,7 @@ fn transfer(p: &mut Pair, to_a: bool, bytes: &[u8]) -> RegisteredPage { assert_eq!(poll(&written), Some(Ok(()))); match poll(&arrival).unwrap().unwrap() { Arrival::Page(page) => page, - Arrival::Released => panic!("expected a page"), + Arrival::Released | Arrival::Device(_) => panic!("expected a page"), } } @@ -1017,7 +1017,7 @@ fn direct_write_reads_in_place_and_holds_the_source_until_completion() { assert_eq!(Arc::strong_count(&source), 1, "released on completion"); match poll(&arrival).unwrap().unwrap() { Arrival::Page(page) => assert_eq!(page.bytes(), &source[..]), - Arrival::Released => panic!("expected a page"), + Arrival::Released | Arrival::Device(_) => panic!("expected a page"), } finish(p); } @@ -1047,7 +1047,7 @@ fn direct_writes_accept_boxed_and_static_sources() { assert_eq!(Arc::strong_count(&source), 1, "released on completion"); match poll(&arrival).unwrap().unwrap() { Arrival::Page(page) => assert_eq!(page.bytes(), &expected[..]), - Arrival::Released => panic!("expected a page"), + Arrival::Released | Arrival::Device(_) => panic!("expected a page"), } } finish(p); diff --git a/cmd/racer-dataplane/verbs/src/node.rs b/cmd/racer-dataplane/verbs/src/node.rs index 2a3a38456b..23dac0c2b8 100644 --- a/cmd/racer-dataplane/verbs/src/node.rs +++ b/cmd/racer-dataplane/verbs/src/node.rs @@ -23,7 +23,7 @@ use std::{ use futures::task::AtomicWaker; use crate::{ - Endpoint, Error, Guard, GuardOwner, PortInfo, Result, Selection, + DeviceMemory, Endpoint, Error, Guard, GuardOwner, PortInfo, Result, Selection, channel::{self, Completer, Op, op}, ffi::{self, Access, NativeDevice, NativeRegion}, link::{self, Break, Epoch, Granted, Limits, Link, Phase, Role, Source}, @@ -157,9 +157,16 @@ enum Command { done: Completer<()>, }, + AttachDevice { + rail: u16, + memory: DeviceMemory, + done: Completer, + }, + Grant { key: LinkKey, epoch: Epoch, + device: bool, transfer: [u8; 16], length: u32, deadline: Instant, @@ -191,6 +198,9 @@ struct NodeShared { pools: RwLock>>, + /// Device pools by rail, once attached. + device_pools: RwLock>>, + /// Breaks that did not fit in a command queue. breaks: Mutex>, @@ -214,6 +224,7 @@ pub fn node(workers: usize) -> Result<(Vec, OwnerPort)> { let shared = Arc::new(NodeShared { links: RwLock::new(BTreeMap::new()), pools: RwLock::new(BTreeMap::new()), + device_pools: RwLock::new(BTreeMap::new()), breaks: Mutex::new(Vec::new()), closed: AtomicBool::new(false), drained: AtomicBool::new(false), @@ -344,6 +355,7 @@ impl Client { self.submit(|done| Command::Grant { key, epoch, + device: false, transfer, length, deadline, @@ -351,6 +363,41 @@ impl Client { }) } + /// Receiver: like [`Self::grant`], but over a free buffer of the rail's + /// device pool (see [`Self::attach_device`]). The arrival is + /// [`crate::Arrival::Device`]. The fence is the same: the page is handed + /// out only after the peer's write completion and the window's signaled + /// invalidation, so the peer can no longer write it. Fails with + /// `Unavailable` if the rail has no device pool, and `Overloaded` if no + /// device buffer is free. Nothing falls back to host memory. + pub fn grant_device( + &self, + key: LinkKey, + epoch: Epoch, + transfer: [u8; 16], + length: u32, + deadline: Instant, + ) -> Op { + self.submit(|done| Command::Grant { + key, + epoch, + device: true, + transfer, + length, + deadline, + done, + }) + } + + /// Register `memory` (a dma-buf, such as GPU memory) on `rail` as its + /// device pool, split into page buffers of the node's buffer length. + /// Once per rail per round, after [`Self::activate`]. Resolves to the + /// number of device buffers. The memory stays registered until the node + /// drains; it is never read or written by the CPU. + pub fn attach_device(&self, rail: u16, memory: DeviceMemory) -> Op { + self.submit(|done| Command::AttachDevice { rail, memory, done }) + } + /// Sender: write `source` into the peer's grant `tag` with immediate, /// waiting for its `Grant` until `deadline`. Resolves on the send /// completion. Dropping the op does not cancel a posted write. @@ -405,6 +452,13 @@ impl Client { Some(pools.get(&rail)?.count_in(state)) } + /// Device page buffers of `rail` in `state`, for diagnostics. `None` + /// without a device pool. + pub fn device_buffers(&self, rail: u16, state: BufferState) -> Option { + let pools = self.shared.device_pools.try_read().ok()?; + Some(pools.get(&rail)?.count_in(state)) + } + /// Close the node: the owner sends `Close` on ready links, breaks every /// link, and frees everything once no page is held. Does not wait. pub fn close(&self) { @@ -554,7 +608,7 @@ impl Owner { self.shared.owner.register(waker); for device in self.devices.values() { - device.pool.shared().register_owner(waker); + device.pool.register_owner(waker); } } @@ -661,15 +715,21 @@ impl Owner { } => { let _ = done.complete(self.connect(key, epoch, remote, link_id, now)); } + Command::AttachDevice { rail, memory, done } => { + let _ = done.complete(self.attach_device(rail, memory)); + } Command::Grant { key, epoch, + device, transfer, length, deadline, done, } => match self.ready(key, epoch) { - Some((link, pool)) => link.grant(pool, transfer, length, deadline, done, now), + Some((link, pool)) => { + link.grant(pool, device, transfer, length, deadline, done, now) + } None => { let _ = done.complete(Err(Error::Unavailable)); } @@ -729,6 +789,33 @@ impl Owner { } } + /// Register `memory` as `rail`'s device pool. + fn attach_device(&mut self, rail: u16, memory: DeviceMemory) -> Result { + #[cfg(any(test, feature = "simulation"))] + let _simulation = self + .simulation + .as_ref() + .map(crate::simulation::Simulation::enter); + let quota = self.quota.clone().ok_or(Error::InvalidRequest)?; + let state = self.devices.get_mut(&rail).ok_or(Error::InvalidRequest)?; + + if state.pool.device_shared().is_some() { + return Err(Error::InvalidRequest); + } + + let region = NativeRegion::dmabuf(state.device.clone(), memory, quota)?; + state.pool.attach_device(region)?; + // The driver waker reaches the device pool on its next registration. + let shared = state.pool.device_shared().expect("just attached").clone(); + let count = shared.count(); + self.shared + .device_pools + .write() + .unwrap_or_else(PoisonError::into_inner) + .insert(rail, shared); + Ok(count) + } + fn activate(&mut self, config: NodeConfig) -> Result> { if self.settings.is_some() || !self.devices.is_empty() { return Err(Error::InvalidRequest); @@ -1052,6 +1139,11 @@ impl Owner { Err(_) => return, } + match self.shared.device_pools.try_write() { + Ok(mut pools) => pools.clear(), + Err(_) => return, + } + if self.devices.values().any(|d| !d.pool.idle()) { return; } @@ -1155,6 +1247,11 @@ impl Drop for Owner { .write() .unwrap_or_else(PoisonError::into_inner) .clear(); + self.shared + .device_pools + .write() + .unwrap_or_else(PoisonError::into_inner) + .clear(); // Pools leak their regions if pages are still held (see `pool.rs`). self.devices.clear(); } @@ -1169,6 +1266,7 @@ fn fail(command: Command, error: Error) { | Command::Write { done, .. } | Command::Release { done, .. } => drop(done.complete(Err(error))), Command::Grant { done, .. } => drop(done.complete(Err(error))), + Command::AttachDevice { done, .. } => drop(done.complete(Err(error))), Command::Break(..) => {} } } diff --git a/cmd/racer-dataplane/verbs/src/pool.rs b/cmd/racer-dataplane/verbs/src/pool.rs index 56c64aed28..19127544ef 100644 --- a/cmd/racer-dataplane/verbs/src/pool.rs +++ b/cmd/racer-dataplane/verbs/src/pool.rs @@ -14,6 +14,12 @@ //! `Owned` only after the fence (receive completion, then a signaled //! `LOCAL_INV` completion). //! +//! A rail may also hold a device pool: page buffers in caller memory reached +//! through a dma-buf (for example GPU memory). Its buffers use the same +//! states and the same fence, but a fenced device buffer becomes a +//! [`DevicePage`], which has no slice or pointer API: the CPU never reads or +//! copies device memory. +//! //! Page buffers only receive. Writes read caller memory in place through the //! device's implicit on-demand paging region, which every pool holds. //! @@ -35,7 +41,7 @@ use std::{ use futures::task::AtomicWaker; -use crate::{Error, GuardOwner, Result, ffi::NativeRegion}; +use crate::{DeviceMemory, Error, GuardOwner, Result, ffi::NativeRegion}; /// Ownership state of one page buffer. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -98,9 +104,12 @@ fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { /// State shared by the owner and every received page. pub(crate) struct PoolShared { /// Start of the registered memory. Valid while any `Arc` exists (see the - /// module docs). + /// module docs). Null for a device pool. base: *mut u8, + /// The dma-buf owner of a device pool. + memory: Option, + buffer_len: usize, states: Box<[AtomicU8]>, @@ -174,6 +183,24 @@ impl PoolShared { #[derive(Debug, Eq, PartialEq)] pub(crate) struct Buffer { index: u32, + + /// In the device pool. + device: bool, +} + +impl Buffer { + /// True for a device pool buffer. + pub(crate) fn is_device(&self) -> bool { + self.device + } +} + +/// A rail's device pool. See the module docs. +struct DevicePool { + shared: Arc, + + /// `None` only during drop. + region: Option>, } /// The owner's side of one device's page buffers. Not `Send`. @@ -187,6 +214,9 @@ pub(crate) struct Pool { /// every write source. `None` only during drop. implicit: Option>, + /// Device page buffers, once attached. + device: Option, + quota: Arc, } @@ -220,6 +250,7 @@ impl Pool { Ok(Self { shared: Arc::new(PoolShared { base, + memory: None, buffer_len, states: (0..count) .map(|_| AtomicU8::new(BufferState::Free.code())) @@ -230,10 +261,112 @@ impl Pool { }), region: Some(region), implicit: Some(implicit), + device: None, quota, }) } + /// Split the dma-buf `region` (from [`NativeRegion::dmabuf`]) into + /// device page buffers of the host buffer length. At most one per pool. + pub(crate) fn attach_device(&mut self, region: Rc) -> Result<()> { + if self.device.is_some() { + return Err(Error::InvalidRequest); + } + + let Some(memory) = region.memory().cloned() else { + return Err(Error::InvalidConfiguration); + }; + + let buffer_len = self.shared.buffer_len; + let count = region.length() / buffer_len; + if count == 0 || count > u32::MAX as usize { + return Err(Error::InvalidConfiguration); + } + + self.device = Some(DevicePool { + shared: Arc::new(PoolShared { + base: std::ptr::null_mut(), + memory: Some(memory), + buffer_len, + states: (0..count) + .map(|_| AtomicU8::new(BufferState::Free.code())) + .collect(), + free: Mutex::new((0..count as u32).rev().collect()), + owner: AtomicWaker::new(), + }), + region: Some(region), + }); + Ok(()) + } + + /// State shared with device pages, if a device pool is attached. + pub(crate) fn device_shared(&self) -> Option<&Arc> { + self.device.as_ref().map(|d| &d.shared) + } + + /// Register the owner's waker for buffers returned to either pool. + pub(crate) fn register_owner(&self, waker: &Waker) { + self.shared.register_owner(waker); + if let Some(device) = &self.device { + device.shared.register_owner(waker); + } + } + + /// State shared by `buffer`'s pool. + fn shared_of(&self, buffer: &Buffer) -> &Arc { + if buffer.device { + &self + .device + .as_ref() + .expect("device buffer without pool") + .shared + } else { + &self.shared + } + } + + /// The registered region `buffer` lives in. + pub(crate) fn region_of(&self, buffer: &Buffer) -> &Rc { + if buffer.device { + let device = self.device.as_ref().expect("device buffer without pool"); + device.region.as_ref().unwrap() + } else { + self.region() + } + } + + /// Take a free device buffer straight into an owner `state`. `None` if + /// no device pool is attached or none is free. + pub(crate) fn take_device(&self, state: BufferState) -> Option { + assert!(state.owner_held() && state != BufferState::Quarantined); + let device = self.device.as_ref()?; + device.shared.pop(state).map(|index| Buffer { + index, + device: true, + }) + } + + /// Fence done after a write into a device buffer: hand `len` bytes to + /// the caller as a [`DevicePage`]. + pub(crate) fn own_device( + &self, + buffer: Buffer, + len: usize, + ) -> std::result::Result { + if !buffer.device || len == 0 || len > self.shared.buffer_len { + return Err(buffer); + } + + let shared = self.shared_of(&buffer).clone(); + shared.set(buffer.index, BufferState::Owned); + + Ok(DevicePage { + shared, + index: buffer.index, + len, + }) + } + /// State shared with received pages. pub(crate) fn shared(&self) -> &Arc { &self.shared @@ -257,18 +390,21 @@ impl Pool { /// Take a free buffer straight into an owner `state`. pub(crate) fn take(&self, state: BufferState) -> Option { assert!(state.owner_held() && state != BufferState::Quarantined); - self.shared.pop(state).map(|index| Buffer { index }) + self.shared.pop(state).map(|index| Buffer { + index, + device: false, + }) } /// Move an owner-held buffer to another owner `state`. pub(crate) fn set(&self, buffer: &Buffer, state: BufferState) { assert!(state.owner_held()); - self.shared.set(buffer.index, state); + self.shared_of(buffer).set(buffer.index, state); } /// Fence done (or nothing was ever exposed): return to `Free`. pub(crate) fn release(&self, buffer: Buffer) { - self.shared.push(buffer.index); + self.shared_of(&buffer).push(buffer.index); } /// Fence done after a write: hand `len` bytes to the CPU as `Owned`. @@ -277,7 +413,7 @@ impl Pool { buffer: Buffer, len: usize, ) -> std::result::Result { - if len == 0 || len > self.shared.buffer_len { + if buffer.device || len == 0 || len > self.shared.buffer_len { return Err(buffer); } @@ -292,18 +428,26 @@ impl Pool { /// Destroy failed: the buffer is never reused. pub(crate) fn leak(&self, buffer: Buffer) { - self.shared.set(buffer.index, BufferState::Leaked); + self.shared_of(&buffer) + .set(buffer.index, BufferState::Leaked); } /// Received pages still alive outside the owner. pub(crate) fn outstanding(&self) -> usize { Arc::strong_count(&self.shared) - 1 + + self + .device + .as_ref() + .map_or(0, |d| Arc::strong_count(&d.shared) - 1) } /// True if dropping now would deregister the region: no buffer is /// outside `Free`. pub(crate) fn idle(&self) -> bool { - self.outstanding() == 0 && self.shared.count_in(BufferState::Free) == self.shared.count() + let free = |shared: &PoolShared| shared.count_in(BufferState::Free) == shared.count(); + self.outstanding() == 0 + && free(&self.shared) + && self.device.as_ref().is_none_or(|d| free(&d.shared)) } } @@ -316,6 +460,9 @@ impl Drop for Pool { std::mem::forget(self.quota.clone()); std::mem::forget(self.region.take()); std::mem::forget(self.implicit.take()); + if let Some(device) = &mut self.device { + std::mem::forget(device.region.take()); + } } } } @@ -366,6 +513,59 @@ impl Drop for RegisteredPage { } } +/// A received device page: no window covers it and no work request uses it. +/// +/// The bytes are `memory()[offset()..offset() + len()]`, in memory the CPU +/// never maps. Read them with the owner of the memory, such as a GPU kernel, +/// and keep this value alive until that work is done. `Send` and `Sync`. +/// Dropping it, on any thread, returns the buffer to the device pool, after +/// which the next peer write may land there. +pub struct DevicePage { + shared: Arc, + + index: u32, + + len: usize, +} + +impl DevicePage { + /// The memory the peer wrote into. + pub fn memory(&self) -> &DeviceMemory { + self.shared.memory.as_ref().expect("device pool memory") + } + + /// Offset of the first received byte from the start of + /// [`Self::memory`] (after [`crate::DmabufMemory::offset`]). + pub fn offset(&self) -> u64 { + self.index as u64 * self.shared.buffer_len as u64 + } + + /// Number of received bytes. + pub fn len(&self) -> usize { + self.len + } + + /// True if no bytes were received. Never true for a delivered page. + pub fn is_empty(&self) -> bool { + self.len == 0 + } +} + +impl std::fmt::Debug for DevicePage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DevicePage") + .field("index", &self.index) + .field("len", &self.len) + .finish() + } +} + +impl Drop for DevicePage { + fn drop(&mut self) { + self.shared.push(self.index); + } +} + #[cfg(test)] mod tests { use std::{ @@ -542,4 +742,61 @@ mod tests { assert_eq!(sim.live_resources(), 3); } } + + #[test] + /// A device pool shares the buffer states and fence, hands out + /// `DevicePage` instead of bytes, keeps host and device buffers apart, + /// and leaks its region with the host region when not idle. + fn device_pool_buffers_and_pages() { + let sim = Simulation::new(); + let device = self::device(&sim); + let (quota, charge) = lifetime_tests::quota(); + let region = + NativeRegion::with_access(device.clone(), LEN, Access::PAGES, quota.clone()).unwrap(); + let implicit = NativeRegion::implicit(device.clone(), quota.clone()).unwrap(); + let mut pool = Pool::new(region, implicit, LEN, quota).unwrap(); + let memory = crate::simulation::MemfdMemory::new(2 * LEN + 1, 0x9000).unwrap(); + let (quota, _observer) = lifetime_tests::quota(); + assert!(pool.take_device(BufferState::Binding).is_none()); + // Host regions are refused as device pools. + let host = + NativeRegion::with_access(device.clone(), LEN, Access::PAGES, quota.clone()).unwrap(); + assert_eq!(pool.attach_device(host), Err(Error::InvalidConfiguration)); + let region = NativeRegion::dmabuf(device.clone(), memory.clone(), quota.clone()).unwrap(); + pool.attach_device(region.clone()).unwrap(); + assert_eq!(pool.attach_device(region), Err(Error::InvalidRequest)); + let shared = pool.device_shared().unwrap().clone(); + assert_eq!(shared.count(), 2); + let buffer = pool.take_device(BufferState::Binding).unwrap(); + assert!(buffer.is_device()); + assert!(pool.region_of(&buffer).is_device()); + assert_eq!(pool.region_of(&buffer).address(), 0x9000); + pool.set(&buffer, BufferState::Invalidating); + assert_eq!(shared.state(0), BufferState::Invalidating); + assert_eq!(pool.shared().state(0), BufferState::Free, "host untouched"); + // A device buffer never becomes a host page with bytes. + let buffer = pool.own(buffer, 10).unwrap_err(); + let page = pool.own_device(buffer, 10).unwrap(); + assert_eq!((page.offset(), page.len()), (0, 10)); + assert_eq!(shared.state(0), BufferState::Owned); + let second = pool.take_device(BufferState::Binding).unwrap(); + let second = pool.own_device(second, LEN).unwrap(); + assert_eq!(second.offset(), LEN as u64); + assert!(pool.take_device(BufferState::Binding).is_none()); + let host = pool.take(BufferState::Binding).unwrap(); + let host = pool.own_device(host, 1).unwrap_err(); + pool.release(host); + drop(second); + assert!(!pool.idle(), "a device page is outstanding"); + drop(shared); + drop(pool); + // Dropped while a device page lives: both regions leak with the charge. + assert_eq!(page.memory().length(), 2 * LEN + 1); + drop(page); + assert_eq!(charge.get(), 1); + assert!( + Arc::strong_count(&memory) > 1, + "leaked region keeps the owner" + ); + } } diff --git a/cmd/racer-dataplane/verbs/src/simulation.rs b/cmd/racer-dataplane/verbs/src/simulation.rs index e15d0a1204..809303eca7 100644 --- a/cmd/racer-dataplane/verbs/src/simulation.rs +++ b/cmd/racer-dataplane/verbs/src/simulation.rs @@ -16,6 +16,11 @@ //! 5. Check what happened with [`Simulation::trace`]. //! //! Addresses seen on the wire are fake; they are never host pointers. +//! A dma-buf region maps the given descriptor, so a memfd from +//! [`MemfdMemory`] stands in for GPU memory. Its wire address is the iova. +//! Like the C adapter, sends, writes, and receives that name a dma-buf region +//! are rejected, and [`Operation::RegisterDmabuf`] traces and faults its +//! registration. //! Device names must be unique within a discovery view, even for different ports. //! A remote QP must already exist in this fabric before its peer connects to it. //! @@ -110,6 +115,8 @@ pub enum Operation { Register, + RegisterDmabuf, + Deregister, Window, @@ -225,7 +232,7 @@ enum Resource { Region { device: u32, - bytes: Box<[u8]>, + bytes: Memory, address: u64, @@ -297,6 +304,10 @@ const MR_CONTROL: u32 = 2; /// `RDMA_VERBS_MR_REMOTE_WRITE`. const MR_REMOTE_WRITE: u32 = 4; +/// A dma-buf region from `rdma_verbs_register_dmabuf`. Peers write through +/// windows; never a send, write, or receive buffer. +const MR_DEVICE: u32 = 16; + /// An implicit on-demand paging region over this process's address space. /// Offsets are virtual addresses; it is only a local write or send source. const MR_IMPLICIT: u32 = 8; @@ -538,6 +549,7 @@ pub(super) fn api() -> Rc { qp_free, register, register_implicit, + register_dmabuf, deregister, bytes, window, @@ -1370,7 +1382,7 @@ unsafe extern "C" fn register(device: *mut c_void, length: u64, access: u32) -> &h.sim, Resource::Region { device: h.id, - bytes: vec![0; length as usize].into_boxed_slice(), + bytes: Memory::Host(vec![0; length as usize].into_boxed_slice()), address, access, }, @@ -1401,7 +1413,7 @@ unsafe extern "C" fn register_implicit(device: *mut c_void) -> *mut c_void { &h.sim, Resource::Region { device: h.id, - bytes: Box::default(), + bytes: Memory::Host(Box::default()), address: 0, access: MR_IMPLICIT, }, @@ -1413,13 +1425,213 @@ unsafe extern "C" fn register_implicit(device: *mut c_void) -> *mut c_void { raw } -/// Pointer to a region's bytes. Valid until the region is freed. +/// Map a dma-buf as a page region whose wire address is `iova`. Like C, +/// `fd` is not kept: the mapping holds its own reference. +unsafe extern "C" fn register_dmabuf( + device: *mut c_void, + offset: u64, + length: u64, + iova: u64, + fd: c_int, +) -> *mut c_void { + let Some(h) = (unsafe { handle(device) }) else { + return std::ptr::null_mut(); + }; + let mut world = h.sim.world.borrow_mut(); + if world.rejected(Operation::RegisterDmabuf, h.id) { + return std::ptr::null_mut(); + } + let mapped = (length != 0 && fd >= 0 && usize::try_from(length).is_ok()) + .then(|| map(fd, offset, length as usize)) + .flatten(); + let Some(pointer) = mapped else { + world.record(Operation::RegisterDmabuf, h.id, None, -1, false); + return std::ptr::null_mut(); + }; + drop(world); + let raw = allocate( + &h.sim, + Resource::Region { + device: h.id, + bytes: Memory::Mapped { + pointer, + length: length as usize, + }, + address: iova, + access: MR_PAGES | MR_DEVICE, + }, + ); + h.sim + .world + .borrow_mut() + .record(Operation::RegisterDmabuf, h.id, None, 0, false); + raw +} + +/// A region's bytes: owned host memory, or a shared mapping of a dma-buf. +enum Memory { + Host(Box<[u8]>), + + Mapped { pointer: NonNull, length: usize }, +} + +impl std::ops::Deref for Memory { + type Target = [u8]; + + /// The region's bytes. + fn deref(&self) -> &[u8] { + match self { + Self::Host(bytes) => bytes, + // SAFETY: the mapping lives until this value drops. + Self::Mapped { pointer, length } => unsafe { + std::slice::from_raw_parts(pointer.as_ptr(), *length) + }, + } + } +} + +impl std::ops::DerefMut for Memory { + /// The region's bytes, writable. + fn deref_mut(&mut self) -> &mut [u8] { + match self { + Self::Host(bytes) => bytes, + // SAFETY: the mapping lives until this value drops. + Self::Mapped { pointer, length } => unsafe { + std::slice::from_raw_parts_mut(pointer.as_ptr(), *length) + }, + } + } +} + +impl Drop for Memory { + /// Unmap a dma-buf mapping. The owner's memory stays. + fn drop(&mut self) { + if let Self::Mapped { pointer, length } = self { + unsafe { + libc::munmap(pointer.as_ptr().cast(), *length); + } + } + } +} + +/// Map `length` bytes of `fd` from `offset`, shared and writable. +fn map(fd: c_int, offset: u64, length: usize) -> Option> { + let offset = libc::off_t::try_from(offset).ok()?; + let pointer = unsafe { + libc::mmap( + std::ptr::null_mut(), + length, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + fd, + offset, + ) + }; + if pointer == libc::MAP_FAILED { + return None; + } + NonNull::new(pointer.cast()) +} + +/// Host memory standing in for a GPU dma-buf: a memfd the simulation maps. +/// +/// The simulated NIC writes through its own shared mapping, so +/// [`Self::read`] sees exactly what a peer wrote. +pub struct MemfdMemory { + fd: std::os::fd::OwnedFd, + + view: NonNull, + + length: usize, + + iova: u64, +} + +// SAFETY: the mapping is plain shared memory, valid for this value's lifetime. +unsafe impl Send for MemfdMemory {} + +unsafe impl Sync for MemfdMemory {} + +impl MemfdMemory { + /// A zeroed memfd of `length` bytes that windows address from `iova`. + pub fn new(length: usize, iova: u64) -> std::io::Result> { + use std::os::fd::{AsRawFd, FromRawFd}; + + if length == 0 { + return Err(std::io::ErrorKind::InvalidInput.into()); + } + let raw = unsafe { libc::memfd_create(c"rdma-verbs-sim".as_ptr(), libc::MFD_CLOEXEC) }; + if raw < 0 { + return Err(std::io::Error::last_os_error()); + } + let fd = unsafe { std::os::fd::OwnedFd::from_raw_fd(raw) }; + if unsafe { libc::ftruncate(fd.as_raw_fd(), length as libc::off_t) } != 0 { + return Err(std::io::Error::last_os_error()); + } + let view = map(fd.as_raw_fd(), 0, length).ok_or_else(std::io::Error::last_os_error)?; + Ok(Arc::new(Self { + fd, + view, + length, + iova, + })) + } + + /// Copy `length` bytes at `offset` out, as a GPU kernel would read them. + pub fn read(&self, offset: usize, length: usize) -> Vec { + assert!( + offset + .checked_add(length) + .is_some_and(|end| end <= self.length) + ); + unsafe { std::slice::from_raw_parts(self.view.as_ptr().add(offset), length) }.to_vec() + } +} + +impl Drop for MemfdMemory { + /// Unmap this value's view. The memfd closes with `fd`. + fn drop(&mut self) { + unsafe { + libc::munmap(self.view.as_ptr().cast(), self.length); + } + } +} + +impl crate::DmabufMemory for MemfdMemory { + /// The memfd. + fn fd(&self) -> std::os::fd::BorrowedFd<'_> { + use std::os::fd::AsFd; + + self.fd.as_fd() + } + + /// Always 0. + fn offset(&self) -> u64 { + 0 + } + + /// The memfd size. + fn length(&self) -> usize { + self.length + } + + /// The iova given to [`Self::new`]. + fn iova(&self) -> u64 { + self.iova + } +} + +/// Pointer to a host region's bytes. Valid until the region is freed. Null +/// for an implicit or dma-buf region, like C. unsafe extern "C" fn bytes(raw: *mut c_void) -> *mut u8 { let Some(h) = (unsafe { handle(raw) }) else { return std::ptr::null_mut(); }; match h.sim.world.borrow_mut().resources.get_mut(&h.id) { - Some(Resource::Region { bytes, .. }) => bytes.as_mut_ptr(), + Some(Resource::Region { + bytes: Memory::Host(bytes), + .. + }) => bytes.as_mut_ptr(), _ => std::ptr::null_mut(), } } @@ -1506,7 +1718,11 @@ unsafe fn qp_and_region<'a>( let (Some(q), Some(r)) = (unsafe { (handle(qp), handle(region)) }) else { return None; }; - if !Rc::ptr_eq(&q.sim.world, &r.sim.world) { + let device = matches!( + r.sim.world.borrow().resources.get(&r.id), + Some(Resource::Region { access, .. }) if access & MR_DEVICE != 0 + ); + if device || !Rc::ptr_eq(&q.sim.world, &r.sim.world) { q.sim .world .borrow_mut() @@ -2100,6 +2316,57 @@ mod tests { assert_eq!(foreign.live_resources(), 0); } + #[test] + /// A dma-buf region has no CPU mapping and is never a send, write, or + /// receive buffer, in Rust or at the adapter: only windows reach it. + fn dmabuf_regions_reject_cpu_and_copy_paths() { + let sim = Simulation::new(); + { + let link = link(&sim, |_| {}); + let memory = MemfdMemory::new(8192, 0x5000_0000).unwrap(); + let (quota, _) = lifetime_tests::quota(); + let region = + NativeRegion::dmabuf(link.pages.device.clone(), memory.clone(), quota).unwrap(); + assert!(region.is_device() && !region.is_implicit()); + assert!(region.as_ptr().is_null()); + assert_eq!(region.address(), 0x5000_0000); + assert_eq!(region.length(), 8192); + assert_eq!( + link.a.send(wr(1), ®ion, 0, 16), + Err(Error::InvalidRequest) + ); + assert_eq!( + link.a.post_recv(wr(2), ®ion, 0, 16), + Err(Error::InvalidRequest) + ); + assert_eq!( + link.a + .write_imm(wr(3), ®ion, 0, 16, link.pages.address(), 1, 0), + Err(Error::InvalidRequest) + ); + sim.take_trace(); + let q = link.a.raw.as_ptr(); + let qpn = link.a.endpoint.qpn; + unsafe { + assert!(bytes(region.raw.as_ptr()).is_null()); + assert_eq!(send(q, region.raw.as_ptr(), 0, 16, 51), -1); + events(&sim, &[(Operation::Send, qpn, Some(51), -1, false)]); + assert_eq!(post_recv(q, region.raw.as_ptr(), 0, 16, 52), -1); + events(&sim, &[(Operation::PostRecv, qpn, Some(52), -1, false)]); + let target = link.pages.address(); + assert_eq!( + write_imm(q, region.raw.as_ptr(), 0, 16, target, 1, 0, 53), + -1 + ); + events(&sim, &[(Operation::WriteImm, qpn, Some(53), -1, false)]); + } + drop(region); + assert_eq!(Arc::strong_count(&memory), 1, "deregister drops the owner"); + stop_both(&link); + } + assert_eq!(sim.live_resources(), 0); + } + /// Two connected QPs on one device with explicit depths, a page region, /// and a control region. struct Link { diff --git a/cmd/racer-dataplane/verbs/tests/links.rs b/cmd/racer-dataplane/verbs/tests/links.rs index db674dfa2a..5da2706af7 100644 --- a/cmd/racer-dataplane/verbs/tests/links.rs +++ b/cmd/racer-dataplane/verbs/tests/links.rs @@ -15,7 +15,7 @@ use std::{ use rdma_verbs::{ Arrival, BACKOFF_MIN, BufferState, COMMAND_QUEUE, Client, Epoch, Error, Granted, LinkKey, LinkState, NodeConfig, Op, Opened, Owner, SETUP_TIMEOUT, Setup, Source, - simulation::{Device, Fault, Operation, Simulation}, + simulation::{Device, Fault, MemfdMemory, Operation, Simulation}, }; use uring_runtime::environment::{Environment, SimulationClock}; @@ -197,7 +197,7 @@ fn bytes(data: Vec) -> Source { fn page(arrival: rdma_verbs::Result) -> rdma_verbs::RegisteredPage { match arrival.unwrap() { Arrival::Page(page) => page, - Arrival::Released => panic!("released"), + Arrival::Released | Arrival::Device(_) => panic!("expected a host page"), } } @@ -817,3 +817,169 @@ mod inner { } } } + +/// Device pool memory: two buffers plus a tail too small for a third. +fn device_memory() -> Arc { + MemfdMemory::new(2 * LEN + 5, 0x7000_0000).unwrap() +} + +fn device_page(arrival: rdma_verbs::Result) -> rdma_verbs::DevicePage { + match arrival.unwrap() { + Arrival::Device(page) => page, + other => panic!("expected a device page, got {other:?}"), + } +} + +#[test] +/// A device grant lands the peer's write in dma-buf memory with no CPU copy: +/// the bytes are only visible through the memory owner, the page holds its +/// buffer until dropped, and host grants keep using the host pool. +fn device_grants_land_in_dmabuf_memory() { + let mut w = world(); + let (ea, eb) = link_up(&mut w); + let memory = device_memory(); + let attached = w.a.clients[0].attach_device(7, memory.clone()); + assert_eq!(wait(&mut w, &attached), Ok(2)); + assert_eq!(w.a.clients[0].device_buffers(7, BufferState::Free), Some(2)); + assert_eq!(w.b.clients[0].device_buffers(7, BufferState::Free), None); + let deadline = now() + 5 * SECOND; + let mut held = Vec::new(); + for round in 0..2u8 { + let granted = w.a.clients[0].grant_device(TO_B, ea, [round; 16], 100, deadline); + let granted = wait(&mut w, &granted).unwrap(); + let data = vec![0xa0 | round; 100]; + let written = w.b.clients[1].write(TO_A, eb, granted.tag, bytes(data.clone()), deadline); + assert_eq!(wait(&mut w, &written), Ok(())); + let page = device_page(ready(&granted.arrival)); + assert_eq!((page.len(), page.is_empty()), (100, false)); + assert_eq!(page.offset(), u64::from(round) * LEN as u64); + assert!(Arc::ptr_eq( + page.memory(), + &(memory.clone() as rdma_verbs::DeviceMemory) + )); + assert_eq!(memory.read(page.offset() as usize, 100), data); + held.push(page); + } + assert_eq!( + w.a.clients[0].device_buffers(7, BufferState::Owned), + Some(2) + ); + // Both device buffers are held: the next device grant is refused, and + // nothing falls back to host memory. + let refused = w.a.clients[0].grant_device(TO_B, ea, [7; 16], 10, deadline); + assert_eq!(wait(&mut w, &refused).err(), Some(Error::Overloaded)); + assert_eq!(w.a.clients[0].buffers(7, BufferState::Free), Some(BUFFERS)); + // Host grants still use the host pool. + let host = w.a.clients[0].grant(TO_B, ea, [8; 16], 3, deadline); + let host = wait(&mut w, &host).unwrap(); + let written = w.b.clients[0].write(TO_A, eb, host.tag, bytes(vec![1, 2, 3]), deadline); + assert_eq!(wait(&mut w, &written), Ok(())); + assert_eq!(page(ready(&host.arrival)).bytes(), [1, 2, 3]); + // A page dropped on another thread frees its buffer for the next grant, + // and a release returns a device buffer without producing a page. + let page = held.pop().unwrap(); + std::thread::spawn(move || drop(page)).join().unwrap(); + pump(&mut w); + let granted = w.a.clients[0].grant_device(TO_B, ea, [9; 16], 10, deadline); + let granted = wait(&mut w, &granted).unwrap(); + let released = w.b.clients[0].release(TO_A, eb, granted.tag); + assert_eq!(wait(&mut w, &released), Ok(())); + assert!(matches!(ready(&granted.arrival), Ok(Arrival::Released))); + assert_eq!(w.a.clients[0].device_buffers(7, BufferState::Free), Some(1)); + // Shutdown waits for the held device page. + w.a.clients[0].close(); + pump(&mut w); + assert!(!w.a.clients[0].drained()); + drop(held); + pump(&mut w); + assert!(w.a.clients[0].drained()); + assert_eq!( + Arc::strong_count(&memory), + 1, + "the memory owner is released" + ); + shutdown(w); +} + +#[test] +/// Attach is once per activated rail; a device grant without a device pool +/// is unavailable; a failed registration leaves nothing behind and can be +/// retried. +fn device_attach_rules_and_registration_faults() { + let mut w = world(); + let (ea, _) = link_up(&mut w); + let deadline = now() + 5 * SECOND; + let missing = w.a.clients[0].grant_device(TO_B, ea, [1; 16], 10, deadline); + assert_eq!(wait(&mut w, &missing).err(), Some(Error::Unavailable)); + let memory = device_memory(); + let wrong_rail = w.a.clients[0].attach_device(3, memory.clone()); + assert_eq!(wait(&mut w, &wrong_rail), Err(Error::InvalidRequest)); + let live = w.sim.live_resources(); + w.sim.fault(Operation::RegisterDmabuf, Fault::Reject); + let failed = w.a.clients[0].attach_device(7, memory.clone()); + assert_eq!(wait(&mut w, &failed), Err(Error::Unavailable)); + assert_eq!(w.sim.live_resources(), live, "nothing registered stays"); + assert_eq!(Arc::strong_count(&memory), 1); + // Memory smaller than one buffer is refused after registration, and + // the region is deregistered again. + let small = w.a.clients[0].attach_device(7, MemfdMemory::new(LEN - 1, 0).unwrap()); + assert_eq!(wait(&mut w, &small), Err(Error::InvalidConfiguration)); + assert_eq!(w.sim.live_resources(), live); + let attached = w.a.clients[0].attach_device(7, memory.clone()); + assert_eq!(wait(&mut w, &attached), Ok(2)); + assert_eq!(w.sim.live_resources(), live + 1); + let again = w.a.clients[0].attach_device(7, device_memory()); + assert_eq!(wait(&mut w, &again), Err(Error::InvalidRequest)); + let attempts: Vec = w + .sim + .trace() + .iter() + .filter(|e| e.operation == Operation::RegisterDmabuf) + .map(|e| e.result) + .collect(); + assert_eq!(attempts, [-1, 0, 0]); + shutdown(w); + assert_eq!(Arc::strong_count(&memory), 1); +} + +#[test] +/// A link break while a device grant is exposed fails the arrival, and the +/// buffer is reused only after the QP is destroyed (the same fence as host +/// grants). +fn device_grant_break_fences_before_reuse() { + let mut w = world(); + let (ea, eb) = link_up(&mut w); + let attached = w.a.clients[0].attach_device(7, device_memory()); + assert_eq!(wait(&mut w, &attached), Ok(2)); + let deadline = now() + 5 * SECOND; + let granted = w.a.clients[0].grant_device(TO_B, ea, [1; 16], 10, deadline); + let granted = wait(&mut w, &granted).unwrap(); + assert_eq!( + w.a.clients[0].device_buffers(7, BufferState::Exposed), + Some(1) + ); + w.sim + .fault_on(Operation::WriteImm, None, Fault::DuplicateImmediate); + let written = w.b.clients[0].write(TO_A, eb, granted.tag, bytes(vec![1; 10]), deadline); + pump(&mut w); + let _ = ready(&written); + assert_eq!(ready(&granted.arrival).err(), Some(Error::Unavailable)); + assert!(matches!(w.a.clients[0].link(TO_B), LinkState::Backoff(_))); + assert_eq!(w.a.clients[0].device_buffers(7, BufferState::Free), Some(2)); + shutdown(w); +} + +#[test] +/// Failed deregistration leaks the device region with its memory owner and +/// charge, so the memory is never freed while the NIC may write it. +fn failed_deregister_leaks_device_memory_owner() { + let mut w = world(); + link_up(&mut w); + let memory = device_memory(); + let attached = w.a.clients[0].attach_device(7, memory.clone()); + assert_eq!(wait(&mut w, &attached), Ok(2)); + w.sim.reject(Operation::Deregister, None, true); + drop(w.a.owner.take()); + assert!(Arc::strong_count(&memory) > 1, "memory owner leaked"); + assert!(w.a.charge.load(Ordering::Acquire) >= 1, "charge leaked"); +} diff --git a/deploy/racer/config.yaml.tmpl b/deploy/racer/config.yaml.tmpl index 9f6ceff21e..9a4002354c 100644 --- a/deploy/racer/config.yaml.tmpl +++ b/deploy/racer/config.yaml.tmpl @@ -31,6 +31,11 @@ data: # Discovery and authentication of the operator-managed dataplane workload. RACER_DATAPLANE_SERVICE_ACCOUNT: "racer-dataplane" RACER_DAEMONSET_NAME: "racer-dataplane" + # NVIDIA driver libraries for the dataplane: off, env (default), or cdi. + # env sets NVIDIA_VISIBLE_DEVICES=all and NVIDIA_DRIVER_CAPABILITIES=compute,utility. + # cdi also adds the Pod annotation cdi.k8s.io/racer-dataplane: nvidia.com/gpu=all; + # containerd then fails the Pod on any node without an NVIDIA CDI spec. + # RACER_DATAPLANE_NVIDIA: "env" RACER_CREDENTIALS_SECRET_NAME: {{ .CredentialsSecretName | default "racer-credentials" | quote }} RACER_VERSION_CONFIGMAP_NAME: {{ .VersionConfigMapName | default "racer-version" | quote }} RACER_INSTALLATION_CONFIGMAP_NAME: {{ .InstallationConfigMapName | default "racer-installation" | quote }} diff --git a/deploy/racer/dataplane-config.yaml.tmpl b/deploy/racer/dataplane-config.yaml.tmpl index 2215e0612a..ad6c9c85cc 100644 --- a/deploy/racer/dataplane-config.yaml.tmpl +++ b/deploy/racer/dataplane-config.yaml.tmpl @@ -13,6 +13,16 @@ data: # RDMA requires the same nonempty Site and rail on both peers. NUMA is # autodetected; same-rail candidates prefer local, then unknown, then remote. RACER_ENABLE_RDMA: "auto" + # GPU reads need an image built with RACER_NATIVE_CUDA=true. auto turns GPU + # mode off when no usable GPU is found; true fails startup instead. + # RACER_ENABLE_GPU: "auto" + # Cap on ciphertext staging memory per GPU, in bytes: a nonzero multiple of + # 2 MiB, default 256 MiB. GPU reads stage ciphertext here for decryption. + # RACER_GPU_STAGING_BYTES: "268435456" + # GPU memory per GPU that peers' NICs write GPU-read pages into over RDMA + # (GPUDirect RDMA), in bytes: a multiple of 2 MiB, default 256 MiB. 0 turns + # it off. Used only when both GPU reads and RDMA are on. + # RACER_GPU_LANDING_BYTES: "268435456" # No thread cap by default: eligible physical cores, CPU quota, and resource budgets apply. # I/O shards share NUMA-local crypto threads at approximately 2:1, using leftovers: # eight eligible cores on one NUMA node give five I/O plus three crypto threads diff --git a/deploy/racer/embed_test.go b/deploy/racer/embed_test.go index da3d1c2742..b853542acc 100644 --- a/deploy/racer/embed_test.go +++ b/deploy/racer/embed_test.go @@ -27,7 +27,7 @@ func TestMutatingWebhookManifest(t *testing.T) { require.NoError(t, yaml.UnmarshalStrict(data, &config)) require.Equal(t, "MutatingWebhookConfiguration", config.Kind) require.Equal(t, "racer-object-sidecar-injector", config.Name) - require.Len(t, config.Webhooks, 1) + require.Len(t, config.Webhooks, 2) hook := config.Webhooks[0] require.Equal(t, "racer-object-sidecar.racer.unbounded-cloud.io", hook.Name) @@ -61,6 +61,19 @@ func TestMutatingWebhookManifest(t *testing.T) { require.Equal(t, admissionregistrationv1.NeverReinvocationPolicy, *hook.ReinvocationPolicy) require.Equal(t, int32(5), *hook.TimeoutSeconds) require.Equal(t, []string{"v1"}, hook.AdmissionReviewVersions) + + // The client-socket webhook matches its own label and skips pods that + // already carry the injected marker; everything else is identical. + socket := config.Webhooks[1] + require.Equal(t, "racer-client-socket.racer.unbounded-cloud.io", socket.Name) + require.Equal(t, map[string]string{injector.ClientSocketLabel: injector.EnabledValue}, socket.ObjectSelector.MatchLabels) + require.Empty(t, socket.ObjectSelector.MatchExpressions) + require.Len(t, socket.MatchConditions, 1) + require.Equal(t, "not-already-injected", socket.MatchConditions[0].Name) + require.Contains(t, socket.MatchConditions[0].Expression, "'"+injector.ClientSocketInjectedAnnotation+"'") + + socket.Name, socket.ObjectSelector, socket.MatchConditions = hook.Name, hook.ObjectSelector, hook.MatchConditions + require.Equal(t, hook, socket) } func TestControllerExposesWebhookPort(t *testing.T) { diff --git a/deploy/racer/mutating-webhook.yaml.tmpl b/deploy/racer/mutating-webhook.yaml.tmpl index 625e77c2a8..32b4bc5e7f 100644 --- a/deploy/racer/mutating-webhook.yaml.tmpl +++ b/deploy/racer/mutating-webhook.yaml.tmpl @@ -1,7 +1,11 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 # Injects the racer-object native sidecar into pods labeled -# racer.unbounded-cloud.io/object-sidecar=enabled. The operator fills caBundle. +# racer.unbounded-cloud.io/object-sidecar=enabled, and mounts the Racer client +# socket into pods labeled racer.unbounded-cloud.io/client-socket=enabled. +# Label selectors cannot express OR, so each label has its own webhook. Both +# call the same handler, which applies both opt-ins idempotently. +# The operator fills caBundle. apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: @@ -39,3 +43,35 @@ webhooks: sideEffects: None reinvocationPolicy: Never timeoutSeconds: 5 + - name: racer-client-socket.racer.unbounded-cloud.io + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: racer-controller + namespace: {{ .Namespace | default "unbounded-system" }} + path: /mutate-racer-object-sidecar + port: 9443 + rules: + - operations: ["CREATE"] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + scope: Namespaced + objectSelector: + matchLabels: + racer.unbounded-cloud.io/client-socket: enabled + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: [{{ .Namespace | default "unbounded-system" | quote }}] + matchConditions: + - name: not-already-injected + expression: >- + !has(object.metadata.annotations) || + !('racer.unbounded-cloud.io/client-socket-injected' in object.metadata.annotations) + failurePolicy: Fail + matchPolicy: Equivalent + sideEffects: None + reinvocationPolicy: Never + timeoutSeconds: 5 diff --git a/designs/racer-gpudirect.md b/designs/racer-gpudirect.md new file mode 100644 index 0000000000..44fdb546ca --- /dev/null +++ b/designs/racer-gpudirect.md @@ -0,0 +1,373 @@ +# Racer GPUDirect reads + +## Goal + +Let a client on a node read a Racer object straight into its own GPU memory, with +as few copies as possible. vLLM should be able to load model weights from a +ClusterCache through a loader plugin, with no other special code. + +Out of scope for now: + +- AMD GPUs. The design is NVIDIA-only. +- A capability endpoint such as `GET /v1/gpus`. Clients find out what works by + trying it (see [Fallback](#fallback)). + +## Delivery modes + +| Mode | Endpoint | Who moves the bytes | +|---|---|---| +| GPU | `POST /v1/objects//gpu-read` | The dataplane writes the plaintext into a client GPU allocation. | +| Host | `POST /v1/objects//host-read` | The dataplane writes the plaintext into a client memfd. The client copies it to the GPU. | +| Stream | `POST /v1/objects/` | Today's framed stream over the socket. | + +Both new endpoints are RPCs. The client passes the destination with the request +and the dataplane forgets it when the request ends. Nothing is registered for +longer than one request. + +## Wire protocol + +### Request + +``` +POST /v1/objects/<64 hex key>/gpu-read HTTP/1.1 (or /host-read) +Host: racer +Content-Length: 0 +Range: bytes=- optional, as for Get +If-Match: "" optional, as for Get +Racer-Destination-Offset: required +Racer-Destination-Length: required, > 0 +Racer-Allocation-Size: required for gpu-read only +``` + +- The client sends exactly one file descriptor with the request head, as + `SCM_RIGHTS` ancillary data on the same `sendmsg` as the first head byte. +- For `host-read` the descriptor is a regular file opened for writing, normally + a `memfd`. +- For `gpu-read` the descriptor comes from `cuMemExportToShareableHandle` with + `CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR`. `Racer-Allocation-Size` is the size + passed to `cuMemCreate`. +- `Range`, `If-Match`, `Authorization` and `Racer-Metadata` mean the same as for + Get. `Racer-Page-Credits` and `Racer-Byte-Credits` are ignored. + +The dataplane rejects the request with 400, before writing anything, when: + +- no descriptor, or more than one, came with the head; +- a descriptor came with any other request (the descriptor is closed); +- the ancillary data was truncated (`MSG_CTRUNC`); +- a destination header is missing, duplicated, malformed, or + `offset + length` overflows; +- for `host-read`, the descriptor is not a regular file; +- for `gpu-read`, `offset + length > Racer-Allocation-Size`. + +After resolving the range, the dataplane rejects with 400 if the destination +length is smaller than the range length. Range and version errors keep their +Get status codes (412, 416, and so on). + +`gpu-read` returns 501 when the dataplane has no usable GPU, or when the +allocation is on a GPU the dataplane did not discover. + +### Response + +On success the status is 200 with the same headers as a Get, plus +`Racer-Delivery: gpu` or `Racer-Delivery: host`, and `Connection: close`. + +The body uses the Get frame format: 21 bytes, big-endian, +`kind:u8 | number:u64 | offset:u64 | length:u32`. The difference is that page +frames carry no payload: + +- A kind 1 frame means page `number` is fully written. `offset` is the object + offset of the first written byte and `length` is the number of bytes. The + bytes for object offset `o` are at destination offset + `Racer-Destination-Offset + (o - range start)`. +- The dataplane sends a page frame only after the write is complete and visible + to the client. For `gpu-read` that means the CUDA stream that wrote the page + has finished. +- Page frames arrive in page order. The last frame is kind 2, as for Get. +- `Content-Length` is `21 * (pages + 1)`. +- The client sends no credits. + +If the read fails after the head, the dataplane closes the connection without +the kind 2 frame, as for Get. The destination may then hold partial data. + +### Descriptor lifetime + +1. The dataplane receives the descriptor with `recvmsg` and `MSG_CMSG_CLOEXEC`. +2. For `gpu-read` it imports the handle, maps the whole allocation into its own + CUDA context, and grants itself read and write access. +3. It writes pages. +4. When the read ends for any reason, including a client disconnect, it waits + for all GPU work that touches the allocation, then unmaps it, releases the + handle, and closes the descriptor. +5. Only after step 4 does it send the kind 2 frame. + +If a CUDA free or unmap call fails, the dataplane keeps the resource and counts +the leak, as the `verbs` crate does. It never frees memory that a GPU may still +write. + +`host-read` writes with io_uring `Write` at an explicit file offset. It never +maps the client's file, so a client that truncates its memfd cannot crash the +dataplane. + +### Fallback + +Clients try modes in order: GPU, then host, then stream. + +- An older dataplane does not know the new paths and answers 400 or 404. The + kernel discards the descriptor because the old dataplane reads with plain + `recv`. +- A new dataplane without a usable GPU answers `gpu-read` with 501. + +A client remembers an unsupported mode for the socket path until the process +exits: + +- 501 marks the mode unsupported at once. +- 400, 404 and 405 can also mean a bad request, so the client falls back for + that read but marks the mode unsupported only when a later mode then + succeeds. + +A `host-read` refused because the destination is too small falls back to stream +for that read only. + +### Security + +The socket is already the trust boundary (`docs/content/concepts/racer.md`). The +new endpoints do not change who may read what. A descriptor only lets the +dataplane write into memory the client already owns. The dataplane never reads +from client memory and never writes outside +`[offset, offset + length)`. + +## Dataplane changes + +### Receiving descriptors + +The reactor gains an io_uring `RecvMsg` operation with a control buffer. The +client listener reads request heads with it and keeps any received descriptors +as `OwnedFd`s attached to the parsed request. Descriptors that no request +claims are closed. + +### Host mode + +`host-read` reuses the Get read path (`RangeStream`). For each page it writes the +plaintext slice from the `VerifiedPage` into the client's file with io_uring +`Write`, then sends the page frame. + +### GPU mode + +A new crate, `cmd/racer-dataplane/gpu`, follows the `verbs` pattern: + +- A small C shim, `libracer_gpu_cuda.so.1`, wraps the CUDA driver API, NVML and + cuFile. Racer loads it with `dlopen` and checks `racer_gpu_abi()`. +- Calls run on the native helper thread, never on I/O threads. The one + exception is cuFile reads, which run on four reader threads (below). +- A simulation backend backs tests without GPUs. It can inject failures. +- The Cargo feature is `cuda`. Without it, GPU mode is unavailable. + +At startup the dataplane lists GPUs with the CUDA driver API and NVML. It records +each GPU's UUID, PCI address, NUMA node and compute mode. GPUs in +`EXCLUSIVE_PROCESS` or `PROHIBITED` mode are skipped, because the dataplane's +context would block or be refused. `RACER_ENABLE_GPU=auto|true|false` controls +the feature. `auto` disables GPU mode when nothing usable is found. `true` makes +startup fail instead. + +For each page of a `gpu-read`: + +- If another reader still holds the page's plaintext, the dataplane copies it + host to GPU. +- Otherwise it takes the ciphertext (the memory cache keeps only ciphertext) + and decrypts it on the GPU (below). + +### GPU decryption + +Pages use XAES-256-GCM. The CPU does the XAES key derivation for each page and +passes the derived AES-256 key, the 12-byte GCM nonce and the AAD to the GPU. +The kernel: + +1. checks the CRC64 of the ciphertext; +2. computes the GCM tag; +3. only if both match, decrypts into the client's allocation. + +The AES and GHASH code lives in a header that compiles as both C and CUDA, so +the CPU tests can check it against the `crypto` crate. Keys live only in the +dataplane's GPU memory and are wiped after use. + +Ciphertext reaches GPU staging memory owned by the dataplane in one of three ways: + +- **Memory, origin or HTTP peer:** one host-to-GPU copy. A page cached in + memory is staged from there even if it is also on disk. +- **Local disk:** cuFile reads it straight from NVMe (aligned records only, below). +- **RDMA peer:** the NIC writes it into a GPU landing buffer registered with + `ibv_reg_dmabuf_mr`. The dma-buf comes from `cuMemGetHandleForAddressRange`. + +`RACER_GPU_STAGING_BYTES` caps the dataplane's ciphertext staging memory per GPU. + +How the implementation works today, where it adds to or differs from the plan +above: + +- The read path asks for sealed pages only for `gpu-read` + (`RangeStream::set_sealed`, `Fill::acquire_sealed`). Pages whose plaintext + another reader still holds, origin fills (the CPU encrypts them, so it has the + plaintext), and pages another reader already decrypted are still delivered + as plaintext and copied host to GPU. So are pages that a peer selection fans + out to the read window, and pages whose key cannot be derived or whose AAD is + longer than the kernel accepts (4 KiB). +- The XAES key derivation runs on the page's owning I/O thread + (`PageCrypto::derive_page_key`), not on the crypto worker. It is three AES + blocks. +- Disk records read through host memory keep the CPU CRC check of the + existing fill path; the GPU checks the CRC again. Records read with cuFile + are checked only on the GPU, against the CRC in the record header. +- Each GPU has one staging buffer of 16 MiB + 4 KiB slots, allocated on first + use. Two slots are scratch for the helper's current job: host ciphertext, + and the plaintext of a page the range covers only in part. The GPU decrypts + the whole page there (the tag covers all of it), the helper checks the + verdict, then copies only the requested bytes into the client's window. A + whole page decrypts straight into the window. The scratch plaintext is not + wiped; it is dataplane-owned GPU memory that is never read back to the host. +- The other slots hold cuFile reads. The page's owning worker reads the record + header through host memory (`StoreReader::direct_read`), holds the segment + lease, and asks the GPU helper to read the ciphertext into a free slot. The + helper registers the file with cuFile once and hands the read to one of four + reader threads, so several reads run at once while the helper decrypts. In a + test on one NVMe drive, one thread reached about 5 GiB/s and two or more + reached the drive's limit of about 6 GiB/s. When no slot is free, cuFile + fails, or the record is v1, unaligned, or on a simulated descriptor, the + worker reads the page through host memory instead. +- The helper opens the cuFile driver on another thread at startup. Opening it + probes every GPU and can take most of a second, which would otherwise delay + the first disk read. A cuFile driver or registration failure turns cuFile off + until restart. A failed read only drops that file's registration. +- At startup the helper runs a zero-length decrypt on each GPU. A GPU whose + shim has no decrypt kernels, or a staging cap smaller than the two scratch + slots, gets plaintext pages only. +- When the GPU rejects a page (CRC or tag), the read fails without a page frame + and the page's owner drops that copy from memory and disk, as it does after a + failed CPU decrypt. When it accepts a page that came from a peer, the owner + may persist it, as it does after a CPU decrypt. +- RDMA peers write `gpu-read` pages straight into GPU memory (GPUDirect RDMA + landing), described below. Other RDMA pages land in host memory and take + one host-to-GPU copy when a GPU read needs them. + +### GPUDirect RDMA landing + +When GPU decryption and RDMA are both on, each GPU can get one landing +buffer: GPU memory separate from the staging buffer, sized by +`RACER_GPU_LANDING_BYTES` (default 256 MiB per GPU, 0 turns landing off). +The GPU helper allocates it once, exports it once as a dma-buf, and keeps it +until the process exits. At each RDMA activation, every rail is matched with +at most one GPU, and every GPU with at most one rail: a GPU on the rail's +NUMA node first, then any free GPU (`assign_landing` in +`src/rdma/links.rs`). PCI distance is not used; only NUMA nodes are known on +both sides. The rail registers the landing buffer as its `rdma_verbs` device +pool, so the buffer is split into page-sized slots. If allocation, export, or +registration fails, the rail stays host-only. + +A `gpu-read` page then takes this path (`Fill::land_from_peer`): + +1. If this node is not a candidate for the page, has no local copy, and has + a landing buffer on the request's GPU, it sends one CopyOnly request to the + first provider in visit order, carrying the GPU ordinal. +2. When the hop is RDMA and the rail's landing buffer is on that GPU, the + receiver grants a landing slot instead of a host buffer. If no slot is + free, it grants host memory as usual. +3. The sender writes the ciphertext into the slot. After the usual fence, the + receiver checks the signed completion and that the signed response head + describes exactly this page (`landed_response_head`), and the requester + verifies the signature chain against its request binding. +4. The page becomes a sealed page whose ciphertext is in the landing slot. + The GPU checks the tag (there is no CRC: no host bytes were seen) and + decrypts into the client's allocation. + +A landed page is used once. It does not join or create a single flight, is +not published to memory, and is not persisted, because there are no host +bytes to share. The slot is held until the GPU job finishes. If the provider +answers with a host page (no free slot, or HTTP), that page is used once in +the same way, so it is not fetched twice. A miss, any failure, or a landing +buffer on another GPU falls back to the usual path. Host-only nodes and +non-GPU reads do not change. Relays never ask for landing. + +`racer_rdma_pages_landed_total` counts pages that landed, and the GPU +helper's `landed` counter counts landed pages it decrypted. A landed page the +GPU rejects fails the read like any other sealed page; there is no host copy +to drop. + +The GPU memory used for landing is not charged to the host admission +budget. It is bounded by `RACER_GPU_LANDING_BYTES` per GPU. + +Not covered yet: there is no end-to-end `gpu-read` test across two whole +dataplanes, because the test harness has no two-node fixture with simulated +RDMA and a simulated GPU on the client socket. Tests cover each layer +instead: verbs device pools, landing grants over two simulated nodes, the +peer transport over a simulated link, the fill path with a scripted +provider, and GPU decryption from a landing buffer. + +### Disk layout v2 + +The store already writes format version 2 records (see +`cmd/racer-dataplane/src/store.rs`): the header and its SHA-256 digest, then zero +padding up to `header_len`, then the ciphertext, then zero padding up to the +extent. `header_len` and the ciphertext body are rounded up to the record unit. +Readers treat other versions, including the old unpadded v1 layout, as cold +misses. + +cuFile needs the ciphertext to start on a 4 KiB boundary on the device, so the +writer uses a record unit of at least 4 KiB: the slab's direct-I/O extent unit, +raised to 4 KiB when the extent unit is smaller. Readers still accept any +`header_len` that is a multiple of 512 bytes, so records written with a smaller +unit stay readable and are read through host memory. With 4 KiB record units, +every record in a segment starts 4 KiB aligned relative to the segment, and the +ciphertext is 4 KiB aligned on the device when the segment's physical start is +too. The dataplane does not assume this. It computes +the absolute offset for each read and falls back to host memory if the offset is +not aligned. + +## Deployment + +- The dataplane pod is privileged, so it already sees `/dev/nvidia*`. +- CUDA needs the host driver's user-space libraries. The dataplane asks the + NVIDIA runtime to inject them without requesting `nvidia.com/gpu`: + `NVIDIA_VISIBLE_DEVICES=all` and + `NVIDIA_DRIVER_CAPABILITIES=compute,utility`. The operator knob + `RACER_DATAPLANE_NVIDIA=off|env|cdi` (default `env`) controls this. `cdi` + also adds the CDI annotation for `nvidia.com/gpu=all`. It is opt-in because + containerd fails pod creation when a CDI device cannot be resolved, which + would break the DaemonSet on nodes without NVIDIA GPUs. On other nodes the + environment variables are ignored and GPU mode stays off. +- The image gains a `RACER_NATIVE_CUDA` build argument that builds the shim and + the `cuda` feature. +- The injector gains a label that mounts the client socket directory into app + containers, so a Python client can use it. + +## Clients + +### Python client + +`clients/python/racer` is pure Python. It speaks the socket protocol, including +`SCM_RIGHTS`. torch is needed only for GPU helpers. Through ctypes on `libcuda` +it allocates exportable GPU memory with `cuMemCreate` and the POSIX descriptor +handle type, maps it, and wraps it as a torch tensor. + +### vLLM plugin + +`racer-vllm` registers `--load-format racer` through the +`vllm.general_plugins` entry point and `register_model_loader`. For each +safetensors file it reads the header, reads each tensor's bytes into a reusable +GPU staging buffer, and passes tensor views to `model.load_weights`. Files are +found through `model.safetensors.index.json`. Object keys follow racer-object's +rule (`internal/racerobject/racerobject.go`). + +## Testing + +- Rust simulation tests: descriptor handling, writes that would leave the + window, disconnect during a write, counted leaks, aligned and unaligned records. +- A CPU test that compiles the GPU AES-GCM header as C and compares it with the + `crypto` crate. +- CI builds and loads the CUDA shim. +- Python tests run against a fake server. +- GPU hardware tests are manual or ignored, like the RDMA hardware tests. + +## Order of work + +1. Descriptor passing, `host-read`, the Python client and the vLLM plugin. +2. GPU discovery, the `gpu` crate, GPU decryption and `gpu-read`. +3. Aligned disk records and cuFile reads. +4. GPUDirect RDMA from peers (landing for `gpu-read`, described above). diff --git a/designs/racer-verbs.md b/designs/racer-verbs.md index 786148efb3..75c06e963c 100644 --- a/designs/racer-verbs.md +++ b/designs/racer-verbs.md @@ -112,6 +112,35 @@ thread and returns the buffer to the pool. Dropping a pool with held, quarantined, or leaked buffers keeps the region and charge alive rather than deregistering memory prematurely. +## Device page pools + +A rail can also hold one device pool: caller memory that the NIC reaches +through a Linux dma-buf, such as GPU memory. The caller passes an owner that +implements `DmabufMemory` (descriptor, offset, length, and the NIC address of +the first byte) to `Client::attach_device`, once per rail per round, after +activation. The owner thread registers it with `ibv_reg_dmabuf_mr` +(`LOCAL_WRITE | MW_BIND`) and splits it into buffers of the node's buffer +length. The crate does not depend on any GPU library. + +`Client::grant_device` grants a device buffer. It uses the same states, the +same window bind, and the same fence as a host grant: the arrival resolves +only after the peer's write completion and a successful `LOCAL_INV`, so the +peer can no longer write the buffer. The arrival is `Arrival::Device` with a +`DevicePage`, which names the memory owner, the offset, and the length, and +holds the buffer until it is dropped. A device grant fails with `Unavailable` +when the rail has no device pool and `Overloaded` when no device buffer is +free; it never falls back to host memory. A window stays tied to the region +it first binds, so a link makes device windows on first use and keeps them +apart from host windows, at most one per concurrent grant. + +The CPU never maps or copies device memory. `DevicePage` has no byte API, the +region reports a null pointer, and sends, writes, and receives that name a +device region are rejected in Rust, in the C adapter, and in the simulation. +If deregistration fails, the region leaks with its memory owner and the node +guard, so the memory is never freed while the NIC might write it. The +simulation maps a memfd in place of the dma-buf (`MemfdMemory`), and traces +and faults the registration as `Operation::RegisterDmabuf`. + ## Links For a configured local grant limit `g`: @@ -188,7 +217,8 @@ this crate, remains the authoritative page outcome. There is no in-band ## Native ABI and completion checks -The adapter ABI is version 4. Rust requires that version and the current +The adapter ABI is version 5. Version 5 added `rdma_verbs_register_dmabuf` +and the rule that send, write, and receive posts reject dma-buf regions. Rust requires that version and the current symbols; there is no legacy ABI loader. The completion record is 32 bytes: ```c diff --git a/docs/content/guides/racer-vllm.md b/docs/content/guides/racer-vllm.md new file mode 100644 index 0000000000..12a185193f --- /dev/null +++ b/docs/content/guides/racer-vllm.md @@ -0,0 +1,122 @@ +--- +title: "Load vLLM Models from Racer" +weight: 11 +description: "Load model weights from a Racer cache straight into GPU memory with vLLM." +--- + +{{< callout type="warning" >}} +Racer is pre-release. Its APIs, flags, and on-disk formats may change without a +migration path. See the [Racer reference]({{< relref "reference/racer" >}}). +{{< /callout >}} + +This guide loads vLLM model weights from a Racer cache. The `racer` load format +reads each safetensors file through the local Racer dataplane. When GPU mode is +on, the dataplane writes the bytes straight into GPU memory. Otherwise the +client falls back to host memory, then to a plain stream. + +## Before You Start + +- A ClusterCache whose origin holds the model files. The + [Azure Blob guide]({{< relref "guides/azure-blob-cache" >}}) shows how to make + one. This guide uses a cache named `models`, a bucket named `llama`, and the + prefix `Llama-3.1-8B-Instruct`. +- NVIDIA nodes set up by the unbounded agent. + +## 1. Turn On GPU Mode + +Build the dataplane image with CUDA support and push it to the registry the +operator uses: + +```bash +podman build --build-arg RACER_NATIVE_CUDA=true \ + -t /racer-dataplane: -f images/racer-dataplane/Containerfile . +``` + +The CUDA image also installs NVIDIA's cuFile library (`libcufile-12-9`) for +GPUDirect Storage reads, because the NVIDIA container runtime injects only the +driver libraries. cuFile uses its packaged default configuration, except that +Racer turns off cuFile's compatibility mode: without GPUDirect Storage +(`nvidia-fs` or PCIe P2PDMA) on the host, disk reads use Racer's own host +memory path instead of cuFile's slower POSIX fallback. Set +`CUFILE_ALLOW_COMPAT_MODE=true` on the dataplane to keep cuFile's fallback. +cuFile reads the filesystem type from udev, so the dataplane mounts the host's +`/run/udev` read-only. + +GPU mode is `auto` by default. To make the dataplane fail when it finds no GPU, +set it to `true`: + +```bash +kubectl -n unbounded-system patch configmap racer-dataplane-config \ + --type merge -p '{"data":{"RACER_ENABLE_GPU":"true"}}' +``` + +The dataplane gets the NVIDIA driver libraries without requesting a GPU. This +is on by default (`RACER_DATAPLANE_NVIDIA: env` in `racer-config`). See +[GPU reads]({{< relref "reference/racer" >}}#gpu-reads). + +## 2. Deploy vLLM + +Add the client socket label and the cache annotation to the vLLM pod: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: vllm + labels: + racer.unbounded-cloud.io/client-socket: enabled + annotations: + racer.unbounded-cloud.io/client-cache: models +spec: + containers: + - name: vllm + image: + command: + - sh + - -c + - | + racer-vllm-fetch racer://models/llama/Llama-3.1-8B-Instruct /models/llama && + exec vllm serve /models/llama --load-format racer + volumeMounts: + - name: model-config + mountPath: /models + resources: + limits: + nvidia.com/gpu: 1 + volumes: + - name: model-config + emptyDir: {} +``` + +The webhook mounts `/run/racer/models/client` read-only into the container and +sets `RACER_SOCKET` to the socket in it. To limit the mount to some containers, +add `racer.unbounded-cloud.io/client-containers: vllm`. The webhook does not +change init containers, so run `racer-vllm-fetch` in the vLLM container. + +## 3. Install racer-vllm + +Install both Python packages into the vLLM image: + +```bash +pip install ./clients/python/racer ./clients/python/racer-vllm +``` + +`racer-vllm` registers the `racer` load format with vLLM through a plugin entry +point. No other code change is needed. + +## 4. Run + +vLLM reads the model config and tokenizer before it loads weights, so loading +takes two steps: + +1. `racer-vllm-fetch racer://// ` copies + `config.json`, the tokenizer files, and the safetensors index into ``. + It also writes `racer-model.json`, which records the `racer://` location. + The prefix is the directory that holds `model.safetensors.index.json` and + the safetensors files. +2. `vllm serve --load-format racer` reads the weights through Racer. + +See the `racer-vllm` README for loader options such as `staging_bytes`. + +The first load fills the cache from the origin. Later loads on any node read +from the cache. diff --git a/docs/content/reference/racer.md b/docs/content/reference/racer.md index e630b3c3a7..43b95c70c4 100644 --- a/docs/content/reference/racer.md +++ b/docs/content/reference/racer.md @@ -202,6 +202,9 @@ Numeric values are unsigned decimal strings: `268435456`, not `256Mi`. | `RACER_CRYPTO_HELPERS` | Unset (automatic) | Exact helper count, from 1 through the exact I/O count; requires `RACER_IO_WORKERS` | | `RACER_ALLOW_SMT` | `false` | Use eligible physical cores by default; `true` opts into logical CPUs | | `RACER_ENABLE_RDMA` | `auto` | `auto`, `true`, or `false`; see [modes](#rdma) | +| `RACER_ENABLE_GPU` | `auto` | `auto`, `true`, or `false`; see [GPU reads](#gpu-reads) | +| `RACER_GPU_STAGING_BYTES` | `268435456` (256 MiB) | Cap on ciphertext staging memory per GPU, in bytes. Must be a nonzero multiple of 2 MiB. Below about 34 MiB the GPU does not decrypt and the CPU does; see [GPU reads](#gpu-reads) | +| `RACER_GPU_LANDING_BYTES` | `268435456` (256 MiB) | GPU memory per GPU that peers' NICs write GPU-read pages into over RDMA, in bytes. A multiple of 2 MiB; `0` turns it off. Below one page slot (16 MiB plus 4 KiB) it is off; see [GPU reads](#gpu-reads) | | `RACER_SHARES` | `4` | Enrollment proposal; explicit Node `shares` wins | | `RACER_PLAINTEXT_BYTES` / `RACER_CIPHERTEXT_BYTES` | Unset: `67108864` (64 MiB) / `134217728` (128 MiB) per final I/O worker | Independent plaintext/ciphertext admission budgets; an explicit value is node-wide and divided among I/O workers | | `RACER_DIRTY_BYTES` | Unset: `167772160` (160 MiB) per final I/O worker | Dirty-data budget; an explicit value is node-wide | @@ -352,6 +355,50 @@ Older dataplanes send pages out of order unless the client sends before SDKs. Older peers accept only one page request at a time from a node. The dataplane detects this and falls back to one request at a time for that peer. +### Host reads + +A client on the node can ask the dataplane to write an object straight into a +file it owns, such as a `memfd`, instead of streaming it over the socket: + +``` +POST /v1/objects//host-read HTTP/1.1 +Host: racer +Content-Length: 0 +Range: bytes=- optional +If-Match: "" optional +Racer-Destination-Offset: +Racer-Destination-Length: +``` + +- Send exactly one file descriptor with the request, as `SCM_RIGHTS` data on + the same `sendmsg` as the start of the request. It must be a regular file + opened for writing, without `O_APPEND`. +- The bytes at object offset `o` go to file offset + `Racer-Destination-Offset + (o - range start)`. +- The dataplane answers 400 if there is no descriptor, more than one, a + descriptor on any other request, a file that is not regular, or a length + smaller than the range. Range and version errors keep their usual statuses. +- A success has `Racer-Delivery: host`. The body uses the normal frames, but + page frames carry no data: a page frame means that page is already in the + file. The last frame is the usual completion frame. The client sends no + credits. +- If the read fails partway, the connection closes without the completion frame + and the file may hold partial data. + +`POST /v1/objects//gpu-read` writes into GPU memory. It works like +`host-read`, with these differences: + +- The descriptor comes from `cuMemExportToShareableHandle` with the POSIX file + descriptor handle type. `Racer-Allocation-Size` is required and must be the + size passed to `cuMemCreate`. +- The answer is 501 when GPU mode is off or the memory is on a GPU the + dataplane did not find. It is 400 when the size does not match the memory or + the window does not fit in it. +- A success has `Racer-Delivery: gpu`. A page frame means the GPU copy of that + page is complete. +- When the read ends for any reason, the dataplane waits for its GPU work, + then releases the memory. See [GPU reads](#gpu-reads). + ### Warm and hard retention Send `HEAD` on the client socket with `Cache-Control: max-age=` to request @@ -677,6 +724,7 @@ also require matching Services, probes, network policy, and monitoring configura | `RACER_PEER_PORT` | `8082` | Managed dataplane peer listener and membership publication; 1024..65535 | | `RACER_DIAGNOSTICS_PORT` | Unset | Managed dataplane defaults to 9090, or 9091 if peer port is 9090; explicit port must be 1024..65535 and distinct | | `RACER_HOST_NETWORK` | `false` | Explicit Racer host-network opt-in; not implied by RDMA | +| `RACER_DATAPLANE_NVIDIA` | `env` | NVIDIA driver library injection for the dataplane: `off`, `env`, or `cdi`; see [GPU reads](#gpu-reads) | | `RACER_POD_NETWORK_NODES` | Unset | Not supported by operator-managed installations; the controller only enrolls Pods of DaemonSet `racer-dataplane`, so a nonempty value fails reconciliation | | `RACER_SNAPSHOT_MAX_AGE` | `30s` | Maximum replicated snapshot age | | `RACER_CERTIFICATE_LIFETIME` | `24h` | Dataplane leaf lifetime; 2 minutes..24 hours | @@ -933,6 +981,118 @@ for server limits, partial failures, and restart behavior. For tests, `github.com/Azure/unbounded/pkg/racersdk/racersdktest` runs a noncaching fake Racer. It does not test distributed or RDMA behavior. +## Client socket mount + +Pods that talk to the dataplane socket directly, such as the Python client, +opt in with a label. The admission webhook then mounts the cache's client +directory read-only and tells the app where the socket is. + +| Key | Meaning | +|-----|---------| +| Label `racer.unbounded-cloud.io/client-socket: enabled` | Turns on the mount | +| Annotation `racer.unbounded-cloud.io/client-cache` | ClusterCache name. Required | +| Annotation `racer.unbounded-cloud.io/client-containers` | Optional comma-separated app container names. Default: every app container. Init containers are never changed | +| Annotation `racer.unbounded-cloud.io/client-socket-injected` | Set by the webhook to the cache name; do not edit | + +Each selected container gets: + +- `/run/racer//client` mounted read-only at the same path. A read-only + mount still allows `connect` and `sendmsg` on the socket. +- `RACER_SOCKET=/run/racer//client/socket`. + +The pod is placed on Linux nodes without the `exclude` label, like sidecar pods. +The hostPath type is `Directory`, so the pod waits until the dataplane has +created the cache directory. Host networking is allowed. + +This label works alone or with `object-sidecar`. When both name the same cache, +the pod gets one shared volume. The webhook rejects the pod when the cache name +is missing or invalid, when a listed container does not exist, or when a +container already sets `RACER_SOCKET` or mounts that path. + +## GPU reads + +GPU reads let a client on the node read an object straight into its own NVIDIA +GPU memory. They need: + +- A dataplane image built with `--build-arg RACER_NATIVE_CUDA=true`. +- An NVIDIA node set up by the unbounded agent, which installs the NVIDIA + container runtime and a CDI spec. +- `RACER_ENABLE_GPU` in `racer-dataplane-config`. `auto` turns GPU mode off + when no usable GPU is found, or when the image was built without CUDA. `true` + makes startup fail instead. `false` turns it off. + +At startup the dataplane logs each GPU it finds: ordinal, UUID, PCI address, +NUMA node, compute mode, and whether it can use it. It skips GPUs in +`EXCLUSIVE_PROCESS` or `PROHIBITED` mode. One helper thread does all GPU work +for the process, except cuFile disk reads, which run on four reader threads. At most 256 GPU reads run at once; more get 503. + +Where the bytes come from decides who decrypts them: + +- A page whose plaintext another read still holds in memory is copied to the + GPU. The memory cache itself keeps only encrypted pages. +- A page fetched from the origin is encrypted on the CPU, so its plaintext is + copied to the GPU too. +- Other pages reach the GPU still encrypted, and the GPU checks and decrypts + them. This covers pages kept encrypted in memory, pages on local disk, and + pages from peers. The CPU does not decrypt them. +- A page on local disk is read straight from the disk into GPU memory with + cuFile when it can be. Otherwise the CPU reads it and copies it to the GPU. + This needs GPUDirect Storage on the host (`nvidia-fs`, or PCIe P2PDMA). + `racer_gpu_direct_disk_reads_total` and + `racer_gpu_direct_disk_read_bytes_total` count these reads. After a cuFile + driver or registration failure the dataplane logs + `stage=gpu state=cufile-disabled` and reads through host memory until it + restarts. + +- With RDMA on, a page from a peer can be written by the peer's NIC straight + into GPU memory on this node (GPUDirect RDMA). See below. + +If the GPU finds a damaged page, the read fails and the dataplane drops that +copy from memory or disk, as it does when the CPU finds one. + +When both GPU reads and RDMA are on, each GPU can get a landing buffer of +`RACER_GPU_LANDING_BYTES`, split into page slots. Each RDMA rail gets at most +one GPU's landing buffer, preferring a GPU on the rail's NUMA node. A GPU +read that needs a page from a peer over that rail, on that GPU, asks the peer +to write the encrypted page into a free slot. The GPU then checks and +decrypts it. Such a page is used only by that read: it is not kept in memory, +written to disk, or shared with other reads. When no slot is free, the GPUs +do not match, or anything fails, the page comes through host memory as +before. The dataplane logs `stage=gpu-landing` lines at RDMA activation, and +`racer_rdma_pages_landed_total` counts pages that landed in GPU memory. + +`RACER_GPU_STAGING_BYTES` is the GPU memory the dataplane keeps on each GPU for +encrypted pages. It is split into slots of one page (16 MiB plus 4 KiB). Two +slots are needed for decryption, so a smaller cap means the CPU decrypts. The +other slots hold pages read with cuFile. When all of them are in use, the +dataplane reads the next page through host memory instead. If the image has no +GPU decrypt support, the CPU decrypts every page. + +A GPU read fetches its whole `RACER_RANGE_WINDOW_PAGES` window directly, with +cuFile where it can, so a larger window keeps more disk reads in flight. To +load from fast local disks, raise the window, raise `RACER_GPU_STAGING_BYTES` +to hold every page in flight, and raise the client's reads in flight. Each IO +worker's share of `RACER_PLAINTEXT_BYTES` and `RACER_CIPHERTEXT_BYTES` must +still hold the window plus one page. On one NVMe drive, a window of 8, 1 GiB +of staging, and 8 loader reads loaded a 7B model in vLLM in 2.34 seconds, +against 2.44 seconds with the defaults. That is close to the drive's speed. + +The dataplane does not request `nvidia.com/gpu`. It is privileged, so it +already sees `/dev/nvidia*`. It only needs the host driver libraries. +`RACER_DATAPLANE_NVIDIA` in `racer-config` controls how it asks for them: + +| `RACER_DATAPLANE_NVIDIA` | Behavior | +|--------------------------|----------| +| `env` (default) | Sets `NVIDIA_VISIBLE_DEVICES=all` and `NVIDIA_DRIVER_CAPABILITIES=compute,utility` on the dataplane container. The NVIDIA runtime injects the libraries. Other runtimes ignore these variables, so nodes without NVIDIA are not affected | +| `cdi` | Also adds the Pod annotation `cdi.k8s.io/racer-dataplane: nvidia.com/gpu=all`. containerd fails to create the container on any node that cannot resolve this CDI device, so use it only when every dataplane node has an NVIDIA CDI spec | +| `off` | Adds nothing | + +On nodes set up by the unbounded agent, the NVIDIA runtime is the default +containerd runtime and runs in CDI mode, so `env` is enough there. These env +entries are operator wiring: workload overrides cannot change them. + +See [Load vLLM models from Racer]({{< relref "guides/racer-vllm" >}}). + ## RDMA RDMA is optional per hop. Peers need the same **nonempty Site**, the page's selected rail diff --git a/images/racer-dataplane/Containerfile b/images/racer-dataplane/Containerfile index 463fb131fa..032c96fc65 100644 --- a/images/racer-dataplane/Containerfile +++ b/images/racer-dataplane/Containerfile @@ -7,6 +7,11 @@ ARG RUNTIME_IMAGE=ubuntu:noble # Build on the target platform: both Rust and the verbs adapter use its ABI. FROM ${RUST_IMAGE} AS builder ARG RACER_NATIVE_RDMA=true +# Build the CUDA shim (libracer_gpu_cuda.so.1) and the `cuda` feature. The +# shim dlopens the driver at runtime, so only nvcc (for the decrypt kernel PTX) +# is installed here; the runtime image gets the driver from the NVIDIA runtime. +ARG RACER_NATIVE_CUDA=false +ARG CUDA_NVCC_PACKAGE=cuda-nvcc-12-9 ENV RUSTFLAGS="-C force-frame-pointers=yes" WORKDIR /src @@ -17,6 +22,26 @@ RUN case "$RACER_NATIVE_RDMA" in \ *) echo "RACER_NATIVE_RDMA must be true or false" >&2; exit 1 ;; \ esac +RUN set -eu; \ + case "$RACER_NATIVE_CUDA" in \ + true) \ + case "$(uname -m)" in \ + x86_64) repo=x86_64 ;; \ + aarch64) repo=sbsa ;; \ + *) echo "unsupported CUDA architecture $(uname -m)" >&2; exit 1 ;; \ + esac; \ + curl -fsSL -o /tmp/cuda-keyring.deb \ + "https://developer.download.nvidia.com/compute/cuda/repos/debian12/$repo/cuda-keyring_1.1-1_all.deb"; \ + dpkg -i /tmp/cuda-keyring.deb; \ + rm /tmp/cuda-keyring.deb; \ + apt-get update; \ + apt-get install -y --no-install-recommends "$CUDA_NVCC_PACKAGE"; \ + rm -rf /var/lib/apt/lists/* ;; \ + false) ;; \ + *) echo "RACER_NATIVE_CUDA must be true or false" >&2; exit 1 ;; \ + esac +ENV PATH="${PATH}:/usr/local/cuda/bin" + # Copy only build inputs, never host target trees or deployment credentials. COPY cmd/racer-dataplane/Cargo.toml cmd/racer-dataplane/Cargo.lock ./ COPY cmd/racer-dataplane/src ./src @@ -39,6 +64,9 @@ COPY cmd/racer-dataplane/crypto/src ./crypto/src COPY cmd/racer-dataplane/verbs/Cargo.toml ./verbs/Cargo.toml COPY cmd/racer-dataplane/verbs/src ./verbs/src COPY cmd/racer-dataplane/verbs/native/verbs.c ./verbs/native/verbs.c +COPY cmd/racer-dataplane/gpu/Cargo.toml cmd/racer-dataplane/gpu/build.rs ./gpu/ +COPY cmd/racer-dataplane/gpu/src ./gpu/src +COPY cmd/racer-dataplane/gpu/native ./gpu/native COPY cmd/racer-dataplane/http/Cargo.toml ./http/Cargo.toml COPY cmd/racer-dataplane/http/src ./http/src COPY cmd/racer-dataplane/http/tests ./http/tests @@ -52,14 +80,19 @@ COPY cmd/racer-dataplane/wire-codec/Cargo.toml ./wire-codec/Cargo.toml COPY cmd/racer-dataplane/wire-codec/src ./wire-codec/src COPY cmd/racer-dataplane/wire-codec/tests ./wire-codec/tests COPY images/racer-dataplane/build-native.sh ./build-native.sh +COPY images/racer-dataplane/build-native-cuda.sh ./build-native-cuda.sh COPY images/racer-dataplane/check-debug-info.sh ./check-debug-info.sh RUN set -eu; \ mkdir -p /out/unbounded/bin /out/usr/local/lib; \ + features=; \ + if [ "$RACER_NATIVE_RDMA" = true ]; then features=rdma; fi; \ + if [ "$RACER_NATIVE_CUDA" = true ]; then features="${features:+$features,}cuda"; fi; \ + cargo build --locked --release --bin racer-dataplane --no-default-features ${features:+--features "$features"}; \ if [ "$RACER_NATIVE_RDMA" = true ]; then \ - cargo build --locked --release --bin racer-dataplane --no-default-features --features rdma; \ sh ./build-native.sh verbs/native/verbs.c /out/usr/local/lib/librdma_verbs.so.1; \ - else \ - cargo build --locked --release --bin racer-dataplane --no-default-features; \ + fi; \ + if [ "$RACER_NATIVE_CUDA" = true ]; then \ + RACER_GPU_REQUIRE_PTX=1 sh ./build-native-cuda.sh gpu/native /out/usr/local/lib/libracer_gpu_cuda.so.1; \ fi; \ sh ./check-debug-info.sh target/release/racer-dataplane; \ install -m 0755 target/release/racer-dataplane /out/unbounded/bin/racer-dataplane @@ -70,6 +103,42 @@ RUN set -eu; \ # symbol versions, so the binary and verbs adapter run on noble's newer ones. FROM ${RUNTIME_IMAGE} ARG RACER_NATIVE_RDMA=true +ARG RACER_NATIVE_CUDA=false +# cuFile user-space library for GPUDirect Storage. The NVIDIA container runtime +# injects driver libraries only, not the toolkit's libcufile.so.0, so the CUDA +# image ships it. The repo matches RUNTIME_IMAGE (ubuntu:noble). cuFile loads +# libnuma.so.1.0.0 at runtime to place its buffers; it is not a package +# dependency. +ARG CUDA_CUFILE_PACKAGE=libcufile-12-9 +ARG CUDA_RUNTIME_REPO=ubuntu2404 +RUN set -eu; \ + case "$RACER_NATIVE_CUDA" in \ + true) \ + case "$(uname -m)" in \ + x86_64) repo=x86_64 ;; \ + aarch64) repo=sbsa ;; \ + *) echo "unsupported CUDA architecture $(uname -m)" >&2; exit 1 ;; \ + esac; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates curl; \ + curl -fsSL -o /tmp/cuda-keyring.deb \ + "https://developer.download.nvidia.com/compute/cuda/repos/$CUDA_RUNTIME_REPO/$repo/cuda-keyring_1.1-1_all.deb"; \ + dpkg -i /tmp/cuda-keyring.deb; \ + rm /tmp/cuda-keyring.deb; \ + apt-get update; \ + apt-get install -y --no-install-recommends "$CUDA_CUFILE_PACKAGE" libnuma1; \ + test -e /usr/lib/"$(uname -m)"-linux-gnu/libnuma.so.1.0.0; \ + apt-get purge -y --auto-remove curl; \ + lib=$(dirname "$(find /usr/local -name 'libcufile.so.0' | head -n 1)"); \ + test -n "$lib" && test "$lib" != .; \ + echo "$lib" > /etc/ld.so.conf.d/racer-cufile.conf; \ + ldconfig; \ + ldconfig -p | grep -q 'libcufile.so.0'; \ + apt-get clean; \ + rm -rf /var/lib/apt/lists/* ;; \ + false) ;; \ + *) echo "RACER_NATIVE_CUDA must be true or false" >&2; exit 1 ;; \ + esac RUN set -eu; \ case "$RACER_NATIVE_RDMA" in \ true) set -- libibverbs1 ibverbs-providers ;; \ @@ -85,8 +154,8 @@ RUN set -eu; \ chown -R 65532:65532 /run/racer /var/lib/racer; \ chmod 0700 /var/lib/racer/identity /var/lib/racer/slabs -# Install /unbounded/bin/racer-dataplane and, with RDMA, the verbs adapter -# under /usr/local/lib where the dynamic loader finds it by soname. +# Install /unbounded/bin/racer-dataplane and, with RDMA or CUDA, the verbs +# adapter and CUDA shim under /usr/local/lib where the dynamic loader finds it by soname. COPY --from=builder /out/ / COPY LICENSE NOTICE /unbounded/licenses/ RUN ldconfig diff --git a/images/racer-dataplane/build-native-cuda.sh b/images/racer-dataplane/build-native-cuda.sh new file mode 100755 index 0000000000..46a1bd9718 --- /dev/null +++ b/images/racer-dataplane/build-native-cuda.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +# +# Build libracer_gpu_cuda.so.1 from cmd/racer-dataplane/gpu/native. +# +# The shim dlopens the CUDA driver, NVML and cuFile, so it needs no CUDA +# headers or libraries. When nvcc is found (NVCC, or nvcc on PATH) the decrypt +# kernels are compiled to PTX and embedded; otherwise the shim builds without +# them and GPU decryption reports "no kernels". Set RACER_GPU_REQUIRE_PTX=1 to +# fail instead of building without kernels. +set -eu + +if [ "$#" -ne 2 ]; then + echo "usage: build-native-cuda.sh NATIVE_DIR OUTPUT" >&2 + exit 2 +fi + +native=$1 +output=$2 +mkdir -p "$(dirname "$output")" +nvcc=${NVCC:-$(command -v nvcc || true)} +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +set -- +if [ -n "$nvcc" ]; then + # compute_80 (Ampere) is the PTX baseline; the driver JIT-compiles it for + # the installed GPU. + "$nvcc" -ptx -arch=compute_80 -O3 -std=c++14 -I "$native" \ + -o "$work/gcm_kernel.ptx" "$native/gcm_kernel.cu" + set -- "-DRACER_GPU_PTX=\"$work/gcm_kernel.ptx\"" +elif [ "${RACER_GPU_REQUIRE_PTX:-0}" = 1 ]; then + echo "nvcc not found and RACER_GPU_REQUIRE_PTX=1" >&2 + exit 1 +else + echo "nvcc not found: building libracer_gpu_cuda.so.1 without decrypt kernels" >&2 +fi + +"${CC:-cc}" "$@" -I "$native" \ + -O2 -g -fno-omit-frame-pointer -std=gnu11 -Wall -Wextra -Werror -fPIC -shared \ + -Wl,-soname,libracer_gpu_cuda.so.1 -Wl,-z,defs -Wl,-z,noexecstack \ + -o "$output" "$native/gpu_cuda.c" -ldl -lpthread diff --git a/internal/operator/components/racer/dataplane_test.go b/internal/operator/components/racer/dataplane_test.go index 6073589aa8..b7636f3577 100644 --- a/internal/operator/components/racer/dataplane_test.go +++ b/internal/operator/components/racer/dataplane_test.go @@ -94,6 +94,9 @@ func TestClusterCacheDeploysDataplane(t *testing.T) { require.Equal(t, marker.Data["cluster"], values["RACER_CLUSTER_ID"]) require.Equal(t, "https://racer-controller.custom-system.svc:8443", values["RACER_CONTROL_ENDPOINT"]) require.Equal(t, "[$(RACER_POD_IP)]:8082", values["RACER_PEER_LISTEN"]) + require.Equal(t, "all", values["NVIDIA_VISIBLE_DEVICES"]) + require.Equal(t, "compute,utility", values["NVIDIA_DRIVER_CAPABILITIES"]) + require.NotContains(t, ds.Spec.Template.Annotations, "cdi.k8s.io/racer-dataplane") var bootstrap *corev1.Volume @@ -147,9 +150,11 @@ func TestDataplaneWorkloadInputsFromRacerConfig(t *testing.T) { require.NoError(t, env.Client.Get(t.Context(), objectKey(env, configName), cm)) cm.Data["RACER_HOST_NETWORK"] = "true" cm.Data["RACER_DIAGNOSTICS_PORT"] = "9300" + cm.Data["RACER_DATAPLANE_NVIDIA"] = "cdi" require.NoError(t, env.Client.Update(t.Context(), cm)) persist(t, env, planPass(t, env)) + require.Equal(t, "nvidia.com/gpu=all", liveDataplane(t, env).Spec.Template.Annotations["cdi.k8s.io/racer-dataplane"]) pod := liveDataplane(t, env).Spec.Template.Spec require.True(t, pod.HostNetwork) require.Equal(t, corev1.DNSClusterFirstWithHostNet, pod.DNSPolicy) @@ -159,6 +164,7 @@ func TestDataplaneWorkloadInputsFromRacerConfig(t *testing.T) { {"RACER_POD_NETWORK_NODES", `["node-a"]`, "not supported"}, {"RACER_HOST_NETWORK", "yes", "RACER_HOST_NETWORK"}, {"RACER_DIAGNOSTICS_PORT", "80", "RACER_DIAGNOSTICS_PORT"}, + {"RACER_DATAPLANE_NVIDIA", "auto", "RACER_DATAPLANE_NVIDIA"}, } { t.Run(tc.key, func(t *testing.T) { require.NoError(t, env.Client.Get(t.Context(), objectKey(env, configName), cm)) diff --git a/internal/operator/components/racer/webhook_test.go b/internal/operator/components/racer/webhook_test.go index 851b3c5d16..4480e95317 100644 --- a/internal/operator/components/racer/webhook_test.go +++ b/internal/operator/components/racer/webhook_test.go @@ -29,7 +29,18 @@ func TestWebhookInstalledWithExpectedShape(t *testing.T) { marker := &corev1.ConfigMap{} require.NoError(t, env.Client.Get(t.Context(), objectKey(env, markerName), marker)) require.NoError(t, validateRuntimeOwner(webhook, marker.UID)) - require.Len(t, webhook.Webhooks, 1) + require.Len(t, webhook.Webhooks, 2) + + // The client-socket webhook differs only in its label and match condition. + socketHook := webhook.Webhooks[1] + require.Equal(t, "racer-client-socket.racer.unbounded-cloud.io", socketHook.Name) + require.Equal(t, &metav1.LabelSelector{MatchLabels: map[string]string{"racer.unbounded-cloud.io/client-socket": "enabled"}}, socketHook.ObjectSelector) + require.Len(t, socketHook.MatchConditions, 1) + require.Contains(t, socketHook.MatchConditions[0].Expression, "racer.unbounded-cloud.io/client-socket-injected") + require.Equal(t, webhook.Webhooks[0].ClientConfig, socketHook.ClientConfig) + require.Equal(t, webhook.Webhooks[0].Rules, socketHook.Rules) + require.Equal(t, webhook.Webhooks[0].NamespaceSelector, socketHook.NamespaceSelector) + require.Equal(t, ptr.To(admissionv1.Fail), socketHook.FailurePolicy) hook := webhook.Webhooks[0] require.Equal(t, []admissionv1.RuleWithOperations{{ diff --git a/internal/racer/injector/client_socket_test.go b/internal/racer/injector/client_socket_test.go new file mode 100644 index 0000000000..e71440d5ca --- /dev/null +++ b/internal/racer/injector/client_socket_test.go @@ -0,0 +1,241 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package injector + +import ( + "reflect" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const testClientDir = "/run/racer/" + testCache + "/client" + +// newClientPod returns a pod opted into the client socket only. +func newClientPod(annotations map[string]string) *corev1.Pod { + a := map[string]string{ClientCacheAnnotation: testCache} + + for k, v := range annotations { + if v == "" { + delete(a, k) + continue + } + + a[k] = v + } + + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "ns", Labels: map[string]string{ClientSocketLabel: EnabledValue}, Annotations: a}, + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Name: "setup", Image: "app"}}, + Containers: []corev1.Container{{Name: "app", Image: "app"}, {Name: "helper", Image: "app"}}, + }, + } +} + +func socketMounted(c corev1.Container, volume string) bool { + mount := false + + for _, m := range c.VolumeMounts { + if reflect.DeepEqual(m, corev1.VolumeMount{Name: volume, MountPath: testClientDir, ReadOnly: true}) { + mount = true + } + } + + env := false + + for _, e := range c.Env { + if e.Name == SocketEnv && e.Value == testClientDir+"/socket" { + env = true + } + } + + return mount && env +} + +func TestMutateClientSocketAllContainers(t *testing.T) { + pod := newClientPod(nil) + if err := Mutate(pod, testCfg); err != nil { + t.Fatal(err) + } + + for _, c := range pod.Spec.Containers { + if !socketMounted(c, ClientVolumeName) { + t.Fatalf("container %s: %+v", c.Name, c) + } + } + + if len(pod.Spec.InitContainers) != 1 || len(pod.Spec.InitContainers[0].VolumeMounts) != 0 || len(pod.Spec.InitContainers[0].Env) != 0 { + t.Fatalf("init containers must be untouched and no sidecar added: %+v", pod.Spec.InitContainers) + } + + if len(pod.Spec.Volumes) != 1 { + t.Fatalf("volumes: %+v", pod.Spec.Volumes) + } + + v := pod.Spec.Volumes[0] + if v.Name != ClientVolumeName || v.HostPath.Path != testClientDir || *v.HostPath.Type != corev1.HostPathDirectory { + t.Fatalf("volume: %+v", v) + } + + if pod.Spec.NodeSelector[osLabel] != "linux" || pod.Spec.Affinity == nil { + t.Fatalf("placement: %+v %+v", pod.Spec.NodeSelector, pod.Spec.Affinity) + } + + if pod.Annotations[ClientSocketInjectedAnnotation] != testCache || pod.Annotations[InjectedAnnotation] != "" { + t.Fatalf("annotations: %+v", pod.Annotations) + } + + // A second admission pass must not change anything. + before := pod.DeepCopy() + if err := Mutate(pod, testCfg); err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(before, pod) { + t.Fatal("second Mutate changed the pod") + } +} + +func TestMutateClientSocketSelectedContainers(t *testing.T) { + pod := newClientPod(map[string]string{ClientContainersAnnotation: " helper "}) + if err := Mutate(pod, testCfg); err != nil { + t.Fatal(err) + } + + if socketMounted(pod.Spec.Containers[0], ClientVolumeName) || !socketMounted(pod.Spec.Containers[1], ClientVolumeName) { + t.Fatalf("containers: %+v", pod.Spec.Containers) + } +} + +func TestMutateClientSocketWithSidecarSharesVolume(t *testing.T) { + pod := newPod(nil) + pod.Labels[ClientSocketLabel] = EnabledValue + pod.Annotations[ClientCacheAnnotation] = testCache + + if err := Mutate(pod, testCfg); err != nil { + t.Fatal(err) + } + + if len(pod.Spec.Volumes) != 1 || pod.Spec.Volumes[0].Name != VolumeName { + t.Fatalf("want one shared volume, got %+v", pod.Spec.Volumes) + } + + if len(pod.Spec.InitContainers) != 1 || pod.Spec.InitContainers[0].Name != ContainerName { + t.Fatalf("sidecar missing: %+v", pod.Spec.InitContainers) + } + + if !socketMounted(pod.Spec.Containers[0], VolumeName) { + t.Fatalf("app container: %+v", pod.Spec.Containers[0]) + } + + // Distinct caches get distinct volumes. + pod = newPod(nil) + pod.Labels[ClientSocketLabel] = EnabledValue + pod.Annotations[ClientCacheAnnotation] = "other" + + if err := Mutate(pod, testCfg); err != nil { + t.Fatal(err) + } + + if len(pod.Spec.Volumes) != 2 || pod.Spec.Volumes[1].Name != ClientVolumeName || pod.Spec.Volumes[1].HostPath.Path != "/run/racer/other/client" { + t.Fatalf("volumes: %+v", pod.Spec.Volumes) + } +} + +func TestMutateClientSocketHostNetworkAllowed(t *testing.T) { + pod := newClientPod(nil) + pod.Spec.HostNetwork = true + + if err := Mutate(pod, testCfg); err != nil { + t.Fatal(err) + } +} + +func TestMutateClientSocketNoop(t *testing.T) { + for name, modify := range map[string]func(*corev1.Pod){ + "unlabeled": func(p *corev1.Pod) { p.Labels = nil; p.Annotations = nil }, + "label disabled": func(p *corev1.Pod) { p.Labels[ClientSocketLabel] = "disabled" }, + "already injected": func(p *corev1.Pod) { + p.Annotations = map[string]string{ClientSocketInjectedAnnotation: testCache} + }, + } { + t.Run(name, func(t *testing.T) { + pod := newClientPod(nil) + modify(pod) + before := pod.DeepCopy() + + if err := Mutate(pod, testCfg); err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(before, pod) { + t.Fatal("pod changed") + } + }) + } +} + +func TestMutateClientSocketErrors(t *testing.T) { + for _, tc := range []struct { + name string + anno map[string]string + modify func(*corev1.Pod) + want string + }{ + {name: "missing cache", anno: map[string]string{ClientCacheAnnotation: ""}, want: ClientCacheAnnotation}, + {name: "invalid cache", anno: map[string]string{ClientCacheAnnotation: "../x"}, want: ClientCacheAnnotation}, + {name: "unknown container", anno: map[string]string{ClientContainersAnnotation: "nope"}, want: ClientContainersAnnotation}, + {name: "init container", anno: map[string]string{ClientContainersAnnotation: "setup"}, want: ClientContainersAnnotation}, + {name: "duplicate container", anno: map[string]string{ClientContainersAnnotation: "app,app"}, want: ClientContainersAnnotation}, + {name: "empty list", anno: map[string]string{ClientContainersAnnotation: ""}, want: ClientContainersAnnotation}, + { + name: "volume name taken", + modify: func(p *corev1.Pod) { p.Spec.Volumes = []corev1.Volume{{Name: ClientVolumeName}} }, + want: ClientVolumeName, + }, + { + name: "mount path taken", + modify: func(p *corev1.Pod) { + p.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "x", MountPath: testClientDir}} + }, + want: "already mounts", + }, + { + name: "env taken", + modify: func(p *corev1.Pod) { p.Spec.Containers[1].Env = []corev1.EnvVar{{Name: SocketEnv, Value: "x"}} }, + want: SocketEnv, + }, + { + name: "windows os", + modify: func(p *corev1.Pod) { p.Spec.OS = &corev1.PodOS{Name: corev1.Windows} }, + want: "spec.os.name", + }, + { + name: "windows node selector", + modify: func(p *corev1.Pod) { p.Spec.NodeSelector = map[string]string{osLabel: "windows"} }, + want: osLabel, + }, + } { + t.Run(tc.name, func(t *testing.T) { + pod := newClientPod(tc.anno) + if tc.modify != nil { + tc.modify(pod) + } + + before := pod.DeepCopy() + + err := Mutate(pod, testCfg) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want mention of %q", err, tc.want) + } + + if !reflect.DeepEqual(before, pod) { + t.Fatal("pod changed on error") + } + }) + } +} diff --git a/internal/racer/injector/injector.go b/internal/racer/injector/injector.go index daf46229ad..1f233be29d 100644 --- a/internal/racer/injector/injector.go +++ b/internal/racer/injector/injector.go @@ -2,7 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 // Package injector implements a mutating admission handler that injects the -// racer-object sidecar into opted-in pods as a native (restartable init) sidecar. +// racer-object sidecar into opted-in pods as a native (restartable init) +// sidecar, and mounts the Racer client socket into opted-in app containers. package injector import ( @@ -45,6 +46,22 @@ const ( MemoryLimitAnnotation = "racer.unbounded-cloud.io/object-memory-limit" InjectedAnnotation = "racer.unbounded-cloud.io/object-sidecar-injected" + // ClientSocketLabel opts a pod into a read-only mount of the Racer client + // socket directory, independent of the racer-object sidecar. + ClientSocketLabel = "racer.unbounded-cloud.io/client-socket" + // ClientCacheAnnotation names the ClusterCache whose socket is mounted. + ClientCacheAnnotation = "racer.unbounded-cloud.io/client-cache" + // ClientContainersAnnotation optionally lists, comma-separated, the app + // containers that get the mount. The default is every app container. + ClientContainersAnnotation = "racer.unbounded-cloud.io/client-containers" + // ClientSocketInjectedAnnotation records the mounted cache. + ClientSocketInjectedAnnotation = "racer.unbounded-cloud.io/client-socket-injected" + // ClientVolumeName is the volume added when no volume already mounts the + // cache's client directory. + ClientVolumeName = "racer-client-socket" + // SocketEnv tells clients where the socket is. + SocketEnv = "RACER_SOCKET" + osLabel = "kubernetes.io/os" ) @@ -70,15 +87,26 @@ func (c Config) Validate() error { return nil } -// Mutate injects the racer-object sidecar into pod in place. Pods without the -// opt-in label, or that already contain a container named ContainerName, are -// left unchanged. Opted-in pods must set CacheAnnotation. Missing or invalid -// annotations return an error naming the annotation. +// Mutate applies both opt-ins to pod in place: the racer-object sidecar +// (EnabledLabel) and the client socket mount (ClientSocketLabel). Pods without +// either label are left unchanged. Missing or invalid annotations return an +// error naming the annotation. func Mutate(pod *corev1.Pod, cfg Config) error { if err := cfg.Validate(); err != nil { return err } + if err := mutateSidecar(pod, cfg); err != nil { + return err + } + + return mutateClientSocket(pod) +} + +// mutateSidecar injects the racer-object sidecar. Pods without the opt-in +// label, or that already contain a container named ContainerName, are left +// unchanged. Opted-in pods must set CacheAnnotation. +func mutateSidecar(pod *corev1.Pod, cfg Config) error { if pod.Labels[EnabledLabel] != EnabledValue { return nil } @@ -149,23 +177,166 @@ func Mutate(pod *corev1.Pod, cfg Config) error { }}, }) - if pod.Spec.NodeSelector == nil { - pod.Spec.NodeSelector = map[string]string{} + requireRacerNode(&pod.Spec) + + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + + pod.Annotations[InjectedAnnotation] = cfg.Image + + return nil +} + +// requireRacerNode places the pod on Linux nodes that run the dataplane. +func requireRacerNode(spec *corev1.PodSpec) { + if spec.NodeSelector == nil { + spec.NodeSelector = map[string]string{} + } + + spec.NodeSelector[osLabel] = "linux" + + addExclusionAffinity(spec) +} + +// mutateClientSocket mounts /run/racer//client read-only at the same +// path in the selected app containers and sets SocketEnv. A read-only mount +// still permits connect(2) on the socket: write permission on the socket +// inode is checked, not the mount's read-only flag. Pods without the label, +// or already marked with ClientSocketInjectedAnnotation, are left unchanged. +func mutateClientSocket(pod *corev1.Pod) error { + if pod.Labels[ClientSocketLabel] != EnabledValue { + return nil + } + + if _, done := pod.Annotations[ClientSocketInjectedAnnotation]; done { + return nil + } + + cache, ok := pod.Annotations[ClientCacheAnnotation] + if !ok { + return fmt.Errorf("annotation %s is required and must name the ClusterCache to bind to", ClientCacheAnnotation) + } + + if err := validateCache(cache); err != nil { + return fmt.Errorf("annotation %s has invalid cache name %q", ClientCacheAnnotation, cache) + } + + targets, err := clientTargets(pod) + if err != nil { + return err + } + + if pod.Spec.OS != nil && pod.Spec.OS.Name != corev1.Linux { + return fmt.Errorf("pod spec.os.name %q conflicts with the Racer client socket, which requires linux", pod.Spec.OS.Name) + } + + if v, ok := pod.Spec.NodeSelector[osLabel]; ok && v != "linux" { + return fmt.Errorf("pod nodeSelector %s=%q conflicts with the Racer client socket, which requires linux", osLabel, v) } - pod.Spec.NodeSelector[osLabel] = "linux" + clientDir := "/run/racer/" + cache + "/client" + socket := clientDir + "/socket" - addExclusionAffinity(&pod.Spec) + // Reuse a volume that already mounts this directory, such as the one the + // racer-object sidecar adds for the same cache. + volume := "" + + for _, v := range pod.Spec.Volumes { + if v.HostPath != nil && v.HostPath.Path == clientDir { + volume = v.Name + break + } + } + + if volume == "" { + for _, v := range pod.Spec.Volumes { + if v.Name == ClientVolumeName { + return fmt.Errorf("pod already defines a volume named %q", ClientVolumeName) + } + } + } + + for _, i := range targets { + c := &pod.Spec.Containers[i] + for _, m := range c.VolumeMounts { + if m.MountPath == clientDir { + return fmt.Errorf("container %q already mounts %s", c.Name, clientDir) + } + } + + for _, e := range c.Env { + if e.Name == SocketEnv { + return fmt.Errorf("container %q already sets %s", c.Name, SocketEnv) + } + } + } + + if volume == "" { + volume = ClientVolumeName + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: volume, + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ + Path: clientDir, + Type: ptr.To(corev1.HostPathDirectory), + }}, + }) + } + + for _, i := range targets { + c := &pod.Spec.Containers[i] + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{Name: volume, MountPath: clientDir, ReadOnly: true}) + c.Env = append(c.Env, corev1.EnvVar{Name: SocketEnv, Value: socket}) + } + + requireRacerNode(&pod.Spec) if pod.Annotations == nil { pod.Annotations = map[string]string{} } - pod.Annotations[InjectedAnnotation] = cfg.Image + pod.Annotations[ClientSocketInjectedAnnotation] = cache return nil } +// clientTargets returns indexes into pod.Spec.Containers. +func clientTargets(pod *corev1.Pod) ([]int, error) { + value, ok := pod.Annotations[ClientContainersAnnotation] + if !ok { + targets := make([]int, len(pod.Spec.Containers)) + for i := range targets { + targets[i] = i + } + + return targets, nil + } + + index := make(map[string]int, len(pod.Spec.Containers)) + for i, c := range pod.Spec.Containers { + index[c.Name] = i + } + + var targets []int + + seen := map[string]bool{} + + for _, name := range strings.Split(value, ",") { + name = strings.TrimSpace(name) + + i, found := index[name] + if !found || seen[name] { + return nil, fmt.Errorf("annotation %s must list distinct app container names, separated by commas", ClientContainersAnnotation) + } + + seen[name] = true + + targets = append(targets, i) + } + + return targets, nil +} + type sidecarSpec struct { cache string api string diff --git a/internal/racer/members/members_test.go b/internal/racer/members/members_test.go index 9025ebebf3..ea85878770 100644 --- a/internal/racer/members/members_test.go +++ b/internal/racer/members/members_test.go @@ -172,6 +172,11 @@ func assertWorkloadVolumes(t *testing.T, pod corev1.PodSpec, cfg Config) { t.Fatal("private keys share disposable storage") } + wantUdev := &corev1.HostPathVolumeSource{Path: "/run/udev", Type: ptr.To(corev1.HostPathDirectoryOrCreate)} + if !reflect.DeepEqual(volumes["udev"].HostPath, wantUdev) { + t.Fatal("cuFile needs the host udev database") + } + wantDevices := &corev1.HostPathVolumeSource{Path: "/dev", Type: ptr.To(corev1.HostPathDirectory)} if !reflect.DeepEqual(volumes["devices"].HostPath, wantDevices) { t.Fatal("block device discovery requires the existing host /dev directory") @@ -201,9 +206,10 @@ func assertWorkloadMounts(t *testing.T, container corev1.Container) { "sockets": {Name: "sockets", MountPath: "/run/racer"}, "infiniband": {Name: "infiniband", MountPath: "/dev/infiniband", ReadOnly: true}, "devices": {Name: "devices", MountPath: "/host/dev", ReadOnly: true}, + "udev": {Name: "udev", MountPath: "/run/udev", ReadOnly: true}, } if !reflect.DeepEqual(mounts, wantMounts) { - t.Fatal("mounts must retain token, controller trust, private identity, slabs, sockets, RDMA devices and host devices only") + t.Fatal("mounts must retain token, controller trust, private identity, slabs, sockets, RDMA devices, host devices and udev only") } } @@ -252,7 +258,7 @@ func TestWorkloadNativeRDMAAccess(t *testing.T) { Path: "/dev/infiniband", Type: ptr.To(corev1.HostPathDirectoryOrCreate), }}} - if len(pod.Volumes) != 7 { + if len(pod.Volumes) != 8 { t.Fatalf("unexpected volumes: %v", pod.Volumes) } @@ -303,6 +309,8 @@ func TestWorkloadDataplaneEnvironment(t *testing.T) { "RACER_IDENTITY_DIRECTORY": "/var/lib/racer/identity/private", "RACER_SLAB_DIRECTORY": "/var/lib/racer/slabs", "RACER_DEVICE_DIRECTORY": "/host/dev", + "NVIDIA_VISIBLE_DEVICES": "all", + "NVIDIA_DRIVER_CAPABILITIES": "compute,utility", } if len(env) != len(expected) { t.Fatalf("unexpected configuration: %v", env) @@ -571,7 +579,7 @@ func TestWorkloadConfigIgnoresControllerRuntime(t *testing.T) { } switch key { - case "RACER_PEER_PORT", "RACER_HOST_NETWORK", "RACER_POD_NETWORK_NODES", "RACER_DIAGNOSTICS_PORT", "RACER_DATAPLANE_SERVICE_ACCOUNT", "RACER_DAEMONSET_NAME", "RACER_BOOTSTRAP_TRUST_CONFIGMAP": + case "RACER_PEER_PORT", "RACER_HOST_NETWORK", "RACER_POD_NETWORK_NODES", "RACER_DIAGNOSTICS_PORT", "RACER_DATAPLANE_SERVICE_ACCOUNT", "RACER_DAEMONSET_NAME", "RACER_BOOTSTRAP_TRUST_CONFIGMAP", "RACER_DATAPLANE_NVIDIA": return "", false default: t.Errorf("workload parser requested runtime setting %s", key) diff --git a/internal/racer/workload/workload.go b/internal/racer/workload/workload.go index 161ddc06f1..5a0d99f47b 100644 --- a/internal/racer/workload/workload.go +++ b/internal/racer/workload/workload.go @@ -31,6 +31,31 @@ const ( PodNetworkDaemonSetName = "racer-dataplane-podnet" ) +// NVIDIAInjection selects how the dataplane asks the container runtime for +// the host NVIDIA driver libraries. No mode requests nvidia.com/gpu. +type NVIDIAInjection string + +const ( + // NVIDIAOff adds nothing. + NVIDIAOff NVIDIAInjection = "off" + // NVIDIAEnv sets NVIDIA_VISIBLE_DEVICES and NVIDIA_DRIVER_CAPABILITIES. + // The NVIDIA container runtime acts on them; runc ignores them, so they + // are harmless on nodes without NVIDIA. + NVIDIAEnv NVIDIAInjection = "env" + // NVIDIACDI also adds the CDI pod annotation. containerd fails container + // creation when a CDI device cannot be resolved, so use this only when + // every dataplane node has an NVIDIA CDI spec. + NVIDIACDI NVIDIAInjection = "cdi" +) + +const ( + // CDIAnnotation requests the NVIDIA CDI device on the dataplane Pod. + // containerd treats every key with the cdi.k8s.io/ prefix as a CDI request. + CDIAnnotation = "cdi.k8s.io/racer-dataplane" + // CDIAllGPUs is the fully qualified CDI name for every GPU on the node. + CDIAllGPUs = "nvidia.com/gpu=all" +) + // ManagedNames contains only the explicitly configured workload. func ManagedNames(daemonSetName string) []string { return []string{daemonSetName} @@ -52,6 +77,8 @@ type Config struct { DiagnosticsPort uint16 DataplaneServiceAccount string DaemonSetName string + // Empty selects NVIDIAEnv. + NVIDIA NVIDIAInjection } // ConfigFromLookup reads operator deployment wiring without loading or @@ -75,6 +102,15 @@ func ConfigFromLookup(lookup func(string) (string, bool)) (Config, error) { return Config{}, fmt.Errorf("RACER_HOST_NETWORK must be true or false: %w", wire.InvalidRequest) } + // Unset keeps the zero value, which selects NVIDIAEnv. + var nvidia NVIDIAInjection + if value, ok := lookup("RACER_DATAPLANE_NVIDIA"); ok { + nvidia = NVIDIAInjection(value) + if nvidia != NVIDIAOff && nvidia != NVIDIAEnv && nvidia != NVIDIACDI { + return Config{}, fmt.Errorf("RACER_DATAPLANE_NVIDIA must be off, env, or cdi: %w", wire.InvalidRequest) + } + } + var diagnosticsPort uint64 if value, ok := lookup("RACER_DIAGNOSTICS_PORT"); ok { diagnosticsPort, err = strconv.ParseUint(value, 10, 16) @@ -102,6 +138,7 @@ func ConfigFromLookup(lookup func(string) (string, bool)) (Config, error) { DiagnosticsPort: uint16(diagnosticsPort), DataplaneServiceAccount: env("RACER_DATAPLANE_SERVICE_ACCOUNT", "racer-dataplane"), DaemonSetName: env("RACER_DAEMONSET_NAME", "racer-dataplane"), + NVIDIA: nvidia, } return cfg, cfg.Validate() @@ -116,9 +153,21 @@ func (c Config) Validate() error { return err } + if mode := c.nvidiaMode(); mode != NVIDIAOff && mode != NVIDIAEnv && mode != NVIDIACDI { + return fmt.Errorf("NVIDIA injection must be off, env, or cdi: %w", wire.InvalidRequest) + } + return c.validateEndpoint() } +func (c Config) nvidiaMode() NVIDIAInjection { + if c.NVIDIA == "" { + return NVIDIAEnv + } + + return c.NVIDIA +} + func (c Config) validateNetworkNodes() error { if len(c.PodNetworkNodes) != 0 && (!c.HostNetwork || c.DaemonSetName != DataplaneDaemonSetName) { return fmt.Errorf("pod network exceptions require host networking and the fixed dataplane name: %w", wire.InvalidRequest) @@ -186,6 +235,11 @@ func buildDaemonSet(c Config) *appsv1.DaemonSet { // Use workload identity, not the manager's identity, for the Pod selector. labels := map[string]string{"app.kubernetes.io/name": "racer-dataplane", "app.kubernetes.io/instance": c.DaemonSetName} + var annotations map[string]string + if c.nvidiaMode() == NVIDIACDI { + annotations = map[string]string{CDIAnnotation: CDIAllGPUs} + } + return &appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "DaemonSet"}, ObjectMeta: metav1.ObjectMeta{Name: c.DaemonSetName, Namespace: c.Namespace, Labels: labels}, @@ -200,7 +254,7 @@ func buildDaemonSet(c Config) *appsv1.DaemonSet { }, // Require sustained readiness across probe periods before advancing a rollout. MinReadySeconds: 10, - Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: dataplanePod(c)}, + Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: annotations}, Spec: dataplanePod(c)}, }, } } @@ -310,12 +364,33 @@ func dataplaneVolumes(c Config) []corev1.Volume { volumes = append(volumes, corev1.Volume{Name: mount.name, VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: mount.path, Type: ptr.To(corev1.HostPathDirectoryOrCreate)}}}) } + // cuFile reads the filesystem type of each slab device from the udev + // database and refuses files it cannot classify. + if c.nvidiaMode() != NVIDIAOff { + volumes = append(volumes, corev1.Volume{Name: "udev", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/udev", Type: ptr.To(corev1.HostPathDirectoryOrCreate)}}}) + } + return volumes } func dataplaneContainer(c Config) corev1.Container { diagnosticsPort := diagnosticsPort(c) + mounts := []corev1.VolumeMount{ + {Name: "token", MountPath: "/var/run/racer-token", ReadOnly: true}, + {Name: "bootstrap", MountPath: "/etc/racer/bootstrap", ReadOnly: true}, + {Name: "identity", MountPath: "/var/lib/racer/identity"}, + {Name: "slabs", MountPath: "/var/lib/racer/slabs"}, + {Name: "sockets", MountPath: "/run/racer"}, + // Protect directory entries, not device I/O or the host + // from this privileged container. + {Name: "infiniband", MountPath: "/dev/infiniband", ReadOnly: true}, + {Name: "devices", MountPath: "/host/dev", ReadOnly: true}, + } + if c.nvidiaMode() != NVIDIAOff { + mounts = append(mounts, corev1.VolumeMount{Name: "udev", MountPath: "/run/udev", ReadOnly: true}) + } + return corev1.Container{ Name: "dataplane", Image: c.DataplaneImage, @@ -341,22 +416,12 @@ func dataplaneContainer(c Config) corev1.Container { // implies escalation and all capabilities; do not claim otherwise. Privileged: ptr.To(true), AllowPrivilegeEscalation: ptr.To(true), ReadOnlyRootFilesystem: ptr.To(true), }, - VolumeMounts: []corev1.VolumeMount{ - {Name: "token", MountPath: "/var/run/racer-token", ReadOnly: true}, - {Name: "bootstrap", MountPath: "/etc/racer/bootstrap", ReadOnly: true}, - {Name: "identity", MountPath: "/var/lib/racer/identity"}, - {Name: "slabs", MountPath: "/var/lib/racer/slabs"}, - {Name: "sockets", MountPath: "/run/racer"}, - // Protect directory entries, not device I/O or the host - // from this privileged container. - {Name: "infiniband", MountPath: "/dev/infiniband", ReadOnly: true}, - {Name: "devices", MountPath: "/host/dev", ReadOnly: true}, - }, + VolumeMounts: mounts, } } func dataplaneEnvironment(c Config, diagnosticsPort int32) []corev1.EnvVar { - return []corev1.EnvVar{ + env := []corev1.EnvVar{ { Name: "RACER_CLUSTER_ID", Value: string(c.Cluster), @@ -407,6 +472,17 @@ func dataplaneEnvironment(c Config, diagnosticsPort int32) []corev1.EnvVar { Value: "/host/dev", }, } + + // Ask the NVIDIA runtime for driver libraries only. The privileged + // container already sees /dev/nvidia*; no GPU resource is requested. + if c.nvidiaMode() != NVIDIAOff { + env = append(env, + corev1.EnvVar{Name: "NVIDIA_VISIBLE_DEVICES", Value: "all"}, + corev1.EnvVar{Name: "NVIDIA_DRIVER_CAPABILITIES", Value: "compute,utility"}, + ) + } + + return env } // DesiredDaemonSets builds steady-state placement, not a safe migration plan. diff --git a/internal/racer/workload/workload_test.go b/internal/racer/workload/workload_test.go new file mode 100644 index 0000000000..eccdc7f799 --- /dev/null +++ b/internal/racer/workload/workload_test.go @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package workload + +import ( + "errors" + "testing" + + corev1 "k8s.io/api/core/v1" + + "github.com/Azure/unbounded/internal/racer/wire" +) + +func nvidiaLookup(mode string, set bool) func(string) (string, bool) { + values := map[string]string{ + "RACER_CLUSTER_ID": "00000000-0000-4000-8000-000000000001", + "RACER_CONTROL_URL": "https://racer-controller.unbounded-system.svc:8443", + "RACER_DATAPLANE_IMAGE": "racer-dataplane:test", + } + if set { + values["RACER_DATAPLANE_NVIDIA"] = mode + } + + return func(key string) (string, bool) { + value, ok := values[key] + return value, ok + } +} + +func envValue(c corev1.Container, name string) (string, bool) { + for _, e := range c.Env { + if e.Name == name { + return e.Value, true + } + } + + return "", false +} + +func TestNVIDIAInjectionModes(t *testing.T) { + for _, tc := range []struct { + name string + mode string + set bool + wantEnv bool + wantCDI bool + wantConfig NVIDIAInjection + }{ + {name: "default", wantEnv: true}, + {name: "off", mode: "off", set: true, wantConfig: NVIDIAOff}, + {name: "env", mode: "env", set: true, wantEnv: true, wantConfig: NVIDIAEnv}, + {name: "cdi", mode: "cdi", set: true, wantEnv: true, wantCDI: true, wantConfig: NVIDIACDI}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg, err := ConfigFromLookup(nvidiaLookup(tc.mode, tc.set)) + if err != nil { + t.Fatal(err) + } + + if cfg.NVIDIA != tc.wantConfig { + t.Fatalf("NVIDIA = %q, want %q", cfg.NVIDIA, tc.wantConfig) + } + + daemonSets, err := DesiredDaemonSets(cfg) + if err != nil { + t.Fatal(err) + } + + template := daemonSets[0].Spec.Template + container := template.Spec.Containers[0] + + devices, hasDevices := envValue(container, "NVIDIA_VISIBLE_DEVICES") + caps, hasCaps := envValue(container, "NVIDIA_DRIVER_CAPABILITIES") + + if hasDevices != tc.wantEnv || hasCaps != tc.wantEnv { + t.Fatalf("NVIDIA env present = %v/%v, want %v", hasDevices, hasCaps, tc.wantEnv) + } + + if tc.wantEnv && (devices != "all" || caps != "compute,utility") { + t.Fatalf("NVIDIA env = %q/%q", devices, caps) + } + + value, hasCDI := template.Annotations[CDIAnnotation] + if hasCDI != tc.wantCDI || (hasCDI && value != "nvidia.com/gpu=all") { + t.Fatalf("CDI annotation = %q (present %v), want present %v", value, hasCDI, tc.wantCDI) + } + + // cuFile classifies slab devices through the host udev database. + hasUdevVolume := false + + for _, volume := range template.Spec.Volumes { + if volume.Name == "udev" { + hasUdevVolume = volume.HostPath != nil && volume.HostPath.Path == "/run/udev" + } + } + + hasUdevMount := false + + for _, mount := range container.VolumeMounts { + if mount.Name == "udev" { + hasUdevMount = mount.MountPath == "/run/udev" && mount.ReadOnly + } + } + + if hasUdevVolume != tc.wantEnv || hasUdevMount != tc.wantEnv { + t.Fatalf("udev volume/mount = %v/%v, want %v", hasUdevVolume, hasUdevMount, tc.wantEnv) + } + + // GPU libraries must never come from a GPU resource request. + for _, list := range []corev1.ResourceList{container.Resources.Requests, container.Resources.Limits} { + if _, ok := list["nvidia.com/gpu"]; ok { + t.Fatal("dataplane must not request nvidia.com/gpu") + } + } + }) + } +} + +func TestNVIDIAInjectionRejectsUnknownMode(t *testing.T) { + for _, mode := range []string{"", "true", "auto", "CDI"} { + if _, err := ConfigFromLookup(nvidiaLookup(mode, true)); !errors.Is(err, wire.InvalidRequest) { + t.Fatalf("mode %q: err = %v, want InvalidRequest", mode, err) + } + } + + cfg, err := ConfigFromLookup(nvidiaLookup("", false)) + if err != nil { + t.Fatal(err) + } + + cfg.NVIDIA = "bogus" + if err := cfg.Validate(); !errors.Is(err, wire.InvalidRequest) { + t.Fatalf("Validate = %v, want InvalidRequest", err) + } + + // The zero value keeps the env default for programmatic callers. + cfg.NVIDIA = "" + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } +}