From 010a08aad4d050537326c9a269be55c0c01b37a0 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 18:20:29 +0000 Subject: [PATCH 01/87] docs(racer): design GPUDirect reads --- designs/racer-gpudirect.md | 244 +++++++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 designs/racer-gpudirect.md diff --git a/designs/racer-gpudirect.md b/designs/racer-gpudirect.md new file mode 100644 index 000000000..163817311 --- /dev/null +++ b/designs/racer-gpudirect.md @@ -0,0 +1,244 @@ +# Racer GPUDirect reads + +## Goal + +Let a client on a node read a Racer object straight into its own GPU memory, with +as few copies as possible. vLLM should be able to load model weights from a +ClusterCache through a loader plugin, with no other special code. + +Out of scope for now: + +- AMD GPUs. The design is NVIDIA-only. +- A capability endpoint such as `GET /v1/gpus`. Clients find out what works by + trying it (see [Fallback](#fallback)). + +## Delivery modes + +| Mode | Endpoint | Who moves the bytes | +|---|---|---| +| GPU | `POST /v1/objects//gpu-read` | The dataplane writes the plaintext into a client GPU allocation. | +| Host | `POST /v1/objects//host-read` | The dataplane writes the plaintext into a client memfd. The client copies it to the GPU. | +| Stream | `POST /v1/objects/` | Today's framed stream over the socket. | + +Both new endpoints are RPCs. The client passes the destination with the request +and the dataplane forgets it when the request ends. Nothing is registered for +longer than one request. + +## Wire protocol + +### Request + +``` +POST /v1/objects/<64 hex key>/gpu-read HTTP/1.1 (or /host-read) +Host: racer +Content-Length: 0 +Range: bytes=- optional, as for Get +If-Match: "" optional, as for Get +Racer-Destination-Offset: required +Racer-Destination-Length: required, > 0 +Racer-Allocation-Size: required for gpu-read only +``` + +- The client sends exactly one file descriptor with the request head, as + `SCM_RIGHTS` ancillary data on the same `sendmsg` as the first head byte. +- For `host-read` the descriptor is a regular file opened for writing, normally + a `memfd`. +- For `gpu-read` the descriptor comes from `cuMemExportToShareableHandle` with + `CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR`. `Racer-Allocation-Size` is the size + passed to `cuMemCreate`. +- `Range`, `If-Match`, `Authorization` and `Racer-Metadata` mean the same as for + Get. `Racer-Page-Credits` and `Racer-Byte-Credits` are ignored. + +The dataplane rejects the request with 400, before writing anything, when: + +- no descriptor, or more than one, came with the head; +- a descriptor came with any other request (the descriptor is closed); +- the ancillary data was truncated (`MSG_CTRUNC`); +- a destination header is missing, duplicated, malformed, or + `offset + length` overflows; +- for `host-read`, the descriptor is not a regular file; +- for `gpu-read`, `offset + length > Racer-Allocation-Size`. + +After resolving the range, the dataplane rejects with 400 if the destination +length is smaller than the range length. Range and version errors keep their +Get status codes (412, 416, and so on). + +`gpu-read` returns 501 when the dataplane has no usable GPU, or when the +allocation is on a GPU the dataplane did not discover. + +### Response + +On success the status is 200 with the same headers as a Get, plus +`Racer-Delivery: gpu` or `Racer-Delivery: host`, and `Connection: close`. + +The body uses the Get frame format: 21 bytes, big-endian, +`kind:u8 | number:u64 | offset:u64 | length:u32`. The difference is that page +frames carry no payload: + +- A kind 1 frame means page `number` is fully written. `offset` is the object + offset of the first written byte and `length` is the number of bytes. The + bytes for object offset `o` are at destination offset + `Racer-Destination-Offset + (o - range start)`. +- The dataplane sends a page frame only after the write is complete and visible + to the client. For `gpu-read` that means the CUDA stream that wrote the page + has finished. +- Page frames arrive in page order. The last frame is kind 2, as for Get. +- `Content-Length` is `21 * (pages + 1)`. +- The client sends no credits. + +If the read fails after the head, the dataplane closes the connection without +the kind 2 frame, as for Get. The destination may then hold partial data. + +### Descriptor lifetime + +1. The dataplane receives the descriptor with `recvmsg` and `MSG_CMSG_CLOEXEC`. +2. For `gpu-read` it imports the handle, maps the whole allocation into its own + CUDA context, and grants itself read and write access. +3. It writes pages. +4. When the read ends for any reason, including a client disconnect, it waits + for all GPU work that touches the allocation, then unmaps it, releases the + handle, and closes the descriptor. +5. Only after step 4 does it send the kind 2 frame. + +If a CUDA free or unmap call fails, the dataplane keeps the resource and counts +the leak, as the `verbs` crate does. It never frees memory that a GPU may still +write. + +`host-read` writes with io_uring `Write` at an explicit file offset. It never +maps the client's file, so a client that truncates its memfd cannot crash the +dataplane. + +### Fallback + +Clients try modes in order: GPU, then host, then stream. + +- An older dataplane does not know the new paths and answers 400 or 404. The + kernel discards the descriptor because the old dataplane reads with plain + `recv`. +- A new dataplane without a usable GPU answers `gpu-read` with 501. + +A client treats 400, 404, 405 and 501 on its first attempt at a mode as "mode +not supported" and remembers that for the socket path until the process exits. + +### Security + +The socket is already the trust boundary (`docs/content/concepts/racer.md`). The +new endpoints do not change who may read what. A descriptor only lets the +dataplane write into memory the client already owns. The dataplane never reads +from client memory and never writes outside +`[offset, offset + length)`. + +## Dataplane changes + +### Receiving descriptors + +The reactor gains an io_uring `RecvMsg` operation with a control buffer. The +client listener reads request heads with it and keeps any received descriptors +as `OwnedFd`s attached to the parsed request. Descriptors that no request +claims are closed. + +### Host mode + +`host-read` reuses the Get read path (`RangeStream`). For each page it writes the +plaintext slice from the `VerifiedPage` into the client's file with io_uring +`Write`, then sends the page frame. + +### GPU mode + +A new crate, `cmd/racer-dataplane/gpu`, follows the `verbs` pattern: + +- A small C shim, `libracer_gpu_cuda.so.1`, wraps the CUDA driver API, NVML and + cuFile. Racer loads it with `dlopen` and checks `racer_gpu_abi()`. +- Calls run on the native helper thread, never on I/O threads. +- A simulation backend backs tests without GPUs. It can inject failures. +- The Cargo feature is `cuda`. Without it, GPU mode is unavailable. + +At startup the dataplane lists GPUs with the CUDA driver API and NVML. It records +each GPU's UUID, PCI address, NUMA node and compute mode. GPUs in +`EXCLUSIVE_PROCESS` or `PROHIBITED` mode are skipped, because the dataplane's +context would block or be refused. `RACER_ENABLE_GPU=auto|true|false` controls +the feature. `auto` disables GPU mode when nothing usable is found. `true` makes +startup fail instead. + +For each page of a `gpu-read`: + +- If the plaintext is in memory, the dataplane copies it host to GPU. +- Otherwise it fetches the ciphertext and decrypts it on the GPU (below). + +### GPU decryption + +Pages use XAES-256-GCM. The CPU does the XAES key derivation for each page and +passes the derived AES-256 key, the 12-byte GCM nonce and the AAD to the GPU. +The kernel: + +1. checks the CRC64 of the ciphertext; +2. computes the GCM tag; +3. only if both match, decrypts into the client's allocation. + +The AES and GHASH code lives in a header that compiles as both C and CUDA, so +the CPU tests can check it against the `crypto` crate. Keys live only in the +dataplane's GPU memory and are wiped after use. + +Ciphertext reaches GPU staging memory owned by the dataplane in one of three ways: + +- **Memory, origin or HTTP peer:** one host-to-GPU copy. +- **Local disk:** cuFile reads it straight from NVMe (layout v2 only, below). +- **RDMA peer:** the NIC writes it into GPU memory registered with + `ibv_reg_dmabuf_mr`. The dma-buf comes from `cuMemGetHandleForAddressRange`. + +`RACER_GPU_STAGING_BYTES` caps the dataplane's ciphertext staging memory per GPU. + +### Disk layout v2 + +v1 records put the ciphertext right after a variable-length header, so it is not +aligned. v2 pads the header so the ciphertext starts on a 4 KiB boundary inside +the record. The dataplane writes v2 and still reads v1. Only v2 records can use +cuFile. v1 records are read through host memory. + +## Deployment + +- The dataplane pod is privileged, so it already sees `/dev/nvidia*`. +- CUDA needs the host driver's user-space libraries. The dataplane asks the + NVIDIA runtime to inject them without requesting `nvidia.com/gpu`: + `NVIDIA_VISIBLE_DEVICES=all`, `NVIDIA_DRIVER_CAPABILITIES=compute,utility`, + and the CDI annotation for `nvidia.com/gpu=all`. This relies on the CDI setup + the unbounded agent does on NVIDIA nodes. On other nodes GPU mode stays off. +- The image gains a `RACER_NATIVE_CUDA` build argument that builds the shim and + the `cuda` feature. +- The injector gains a label that mounts the client socket directory into app + containers, so a Python client can use it. + +## Clients + +### Python client + +`clients/python/racer` is pure Python. It speaks the socket protocol, including +`SCM_RIGHTS`. torch is needed only for GPU helpers. Through ctypes on `libcuda` +it allocates exportable GPU memory with `cuMemCreate` and the POSIX descriptor +handle type, maps it, and wraps it as a torch tensor. + +### vLLM plugin + +`racer-vllm` registers `--load-format racer` through the +`vllm.general_plugins` entry point and `register_model_loader`. For each +safetensors file it reads the header, reads each tensor's bytes into a reusable +GPU staging buffer, and passes tensor views to `model.load_weights`. Files are +found through `model.safetensors.index.json`. Object keys follow racer-object's +rule (`internal/racerobject/racerobject.go`). + +## Testing + +- Rust simulation tests: descriptor handling, writes that would leave the + window, disconnect during a write, counted leaks, v1 and v2 records. +- A CPU test that compiles the GPU AES-GCM header as C and compares it with the + `crypto` crate. +- CI builds and loads the CUDA shim. +- Python tests run against a fake server. +- GPU hardware tests are manual or ignored, like the RDMA hardware tests. + +## Order of work + +1. Descriptor passing, `host-read`, the Python client and the vLLM plugin. +2. GPU discovery, the `gpu` crate, GPU decryption and `gpu-read`. +3. Disk layout v2 and cuFile reads. +4. GPUDirect RDMA from peers. From e87c8997257f788f8f530881c9d3c00ac54cda34 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 18:31:10 +0000 Subject: [PATCH 02/87] feat(racer): add pure Python Racer client with host-read and gpu-read --- clients/python/.gitignore | 5 + clients/python/racer/README.md | 57 ++ clients/python/racer/pyproject.toml | 22 + clients/python/racer/src/racer/__init__.py | 48 ++ clients/python/racer/src/racer/_wire.py | 221 +++++++ clients/python/racer/src/racer/client.py | 680 +++++++++++++++++++++ clients/python/racer/src/racer/cuda.py | 377 ++++++++++++ clients/python/racer/src/racer/errors.py | 84 +++ clients/python/racer/src/racer/host.py | 71 +++ clients/python/racer/src/racer/keys.py | 88 +++ clients/python/racer/src/racer/tensors.py | 55 ++ clients/python/racer/tests/conftest.py | 67 ++ clients/python/racer/tests/fakeserver.py | 249 ++++++++ clients/python/racer/tests/test_client.py | 319 ++++++++++ clients/python/racer/tests/test_cuda.py | 51 ++ clients/python/racer/tests/test_keys.py | 63 ++ 16 files changed, 2457 insertions(+) create mode 100644 clients/python/.gitignore create mode 100644 clients/python/racer/README.md create mode 100644 clients/python/racer/pyproject.toml create mode 100644 clients/python/racer/src/racer/__init__.py create mode 100644 clients/python/racer/src/racer/_wire.py create mode 100644 clients/python/racer/src/racer/client.py create mode 100644 clients/python/racer/src/racer/cuda.py create mode 100644 clients/python/racer/src/racer/errors.py create mode 100644 clients/python/racer/src/racer/host.py create mode 100644 clients/python/racer/src/racer/keys.py create mode 100644 clients/python/racer/src/racer/tensors.py create mode 100644 clients/python/racer/tests/conftest.py create mode 100644 clients/python/racer/tests/fakeserver.py create mode 100644 clients/python/racer/tests/test_client.py create mode 100644 clients/python/racer/tests/test_cuda.py create mode 100644 clients/python/racer/tests/test_keys.py diff --git a/clients/python/.gitignore b/clients/python/.gitignore new file mode 100644 index 000000000..c4af65667 --- /dev/null +++ b/clients/python/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.egg-info/ +.pytest_cache/ +build/ +dist/ diff --git a/clients/python/racer/README.md b/clients/python/racer/README.md new file mode 100644 index 000000000..4832d8adb --- /dev/null +++ b/clients/python/racer/README.md @@ -0,0 +1,57 @@ +# racer-client + +Python client for Racer, the node-local object cache in Unbounded. It talks +to the Racer dataplane over its Unix socket. It has no required dependencies. + +```python +import racer + +client = racer.Client(cache="models") # /run/racer/models/client/socket +request = racer.object_request("my-bucket", "llama/config.json") +data = client.read(request) +meta = client.stat(request) +``` + +The socket path is, in order: `socket_path=...`, the `RACER_SOCKET` +environment variable, then `/run/racer//client/socket`. + +## Reading into host memory + +```python +with racer.HostBuffer(meta.size) as buf: + client.read_into_host(request, buf) +``` + +`HostBuffer` is a `memfd`. The client passes it to the dataplane with +`SCM_RIGHTS` (`host-read`), and the dataplane writes straight into it. Older +dataplanes do not know `host-read`; the client then uses the normal stream. + +## Reading into GPU memory + +```python +from racer import cuda, tensors + +alloc = cuda.GpuAllocation(meta.size, device=0) +client.read_into_gpu(request, alloc) +t = tensors.as_tensor(alloc) # uint8 CUDA tensor, no copy +``` + +Modes are tried in order: `gpu-read`, then `host-read` into a pinned staging +buffer plus a copy to the GPU, then the stream plus a copy. When a dataplane +says a mode is not supported, the client remembers that for the socket path +until the process exits. + +`GpuAllocation` uses the CUDA driver API (`libcuda.so.1`, through ctypes). +Set `RACER_LIBCUDA` to load a different library. torch is needed only for +`racer.tensors`. A tensor from `as_tensor` keeps the allocation alive. + +## Object keys + +`object_key(bucket, key, version_id)` gives the same 64-hex key as +racer-object (`internal/racerobject/racerobject.go`). + +## Tests + +``` +make racer-python-test +``` diff --git a/clients/python/racer/pyproject.toml b/clients/python/racer/pyproject.toml new file mode 100644 index 000000000..6c0349f10 --- /dev/null +++ b/clients/python/racer/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[project] +name = "racer-client" +version = "0.1.0" +description = "Python client for the Racer node-local object cache" +readme = "README.md" +license = { text = "Apache-2.0" } +requires-python = ">=3.10" +dependencies = [] + +[project.optional-dependencies] +torch = ["torch"] +test = ["pytest"] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/clients/python/racer/src/racer/__init__.py b/clients/python/racer/src/racer/__init__.py new file mode 100644 index 000000000..8fc7cf085 --- /dev/null +++ b/clients/python/racer/src/racer/__init__.py @@ -0,0 +1,48 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Python client for Racer, the Unbounded node-local object cache.""" + +from .client import ( + GPU, + HOST, + STREAM, + Client, + Metadata, + ReadResult, + forget_modes, + socket_path_for, +) +from .errors import ( + NotFound, + ProtocolError, + RacerError, + StatusError, + TruncatedError, + UnsatisfiableRange, + VersionUnavailable, +) +from .host import HostBuffer +from .keys import Request, canonical_json, object_key, object_request + +__all__ = [ + "GPU", + "HOST", + "STREAM", + "Client", + "HostBuffer", + "Metadata", + "NotFound", + "ProtocolError", + "RacerError", + "ReadResult", + "Request", + "StatusError", + "TruncatedError", + "UnsatisfiableRange", + "VersionUnavailable", + "canonical_json", + "forget_modes", + "object_key", + "object_request", + "socket_path_for", +] diff --git a/clients/python/racer/src/racer/_wire.py b/clients/python/racer/src/racer/_wire.py new file mode 100644 index 000000000..5fa513beb --- /dev/null +++ b/clients/python/racer/src/racer/_wire.py @@ -0,0 +1,221 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Racer client socket wire format. + +This mirrors the Go SDK in pkg/racersdk/internal/wire/wire.go and the +dataplane in cmd/racer-dataplane/object-wire/src/client.rs. +""" + +from __future__ import annotations + +import struct +from dataclasses import dataclass + +from .errors import ProtocolError, RacerError, status_error + +MAX_HEAD_BYTES = 32 * 1024 +MAX_FIELD_BYTES = 8192 +OBJECT_PREFIX = "/v1/objects/" +PAGE_SIZE = 16 * 1024 * 1024 +FRAME_SIZE = 21 +CREDIT_SIZE = 12 +PAGE_FRAME = 1 +COMPLETE_FRAME = 2 +MAX_WIRE_INTEGER = (1 << 63) - 1 + +# Credits the Go SDK grants per Get (pkg/racersdk/client.go: pageCredits = 2). +PAGE_CREDITS = 2 +BYTE_CREDITS = PAGE_CREDITS * PAGE_SIZE + +_FRAME = struct.Struct(">BQQI") +_CREDIT = struct.Struct(">QI") + + +@dataclass(frozen=True) +class Frame: + kind: int + number: int + offset: int + length: int + + +def decode_frame(raw: bytes) -> Frame: + if len(raw) != FRAME_SIZE: + raise ProtocolError("short frame") + return Frame(*_FRAME.unpack(raw)) + + +def encode_frame(kind: int, number: int, offset: int, length: int) -> bytes: + return _FRAME.pack(kind, number, offset, length) + + +def encode_credit(number: int, length: int) -> bytes: + return _CREDIT.pack(number, length) + + +def decode_credit(raw: bytes) -> tuple[int, int]: + return _CREDIT.unpack(raw) + + +def page_count(first: int, end: int) -> int: + if end <= first: + return 0 + return (end - 1) // PAGE_SIZE - first // PAGE_SIZE + 1 + + +class Sequence: + """Validates page and completion frames like wire.Sequence.Accept. + + Pages must arrive strictly in page order, so tracking the count of + delivered pages is enough to reject duplicates. + """ + + def __init__(self, first: int, end: int) -> None: + self.first = first + self.end = end + self.pages = page_count(first, end) + self.delivered = 0 + self.bytes = 0 + self.complete = False + + def accept(self, f: Frame) -> None: + if self.complete: + raise ProtocolError("frame after completion") + if f.kind == COMPLETE_FRAME: + if ( + f.number != self.pages + or self.delivered != self.pages + or f.offset != self.end - self.first + or self.bytes != f.offset + or f.length != 0 + ): + raise ProtocolError("invalid completion frame") + self.complete = True + return + if ( + f.kind != PAGE_FRAME + or f.length == 0 + or f.offset < self.first + or f.offset >= self.end + or f.number != f.offset // PAGE_SIZE + ): + raise ProtocolError("invalid page frame") + start = max(self.first, f.number * PAGE_SIZE) + end = min(self.end, (f.number + 1) * PAGE_SIZE) + if ( + f.offset != start + or f.length != end - start + or f.number != self.first // PAGE_SIZE + self.delivered + ): + raise ProtocolError("page frame out of order") + self.delivered += 1 + self.bytes += f.length + + +def decimal(s: str | None) -> int: + """Parse a canonical non-negative int63 like wire.Decimal.""" + if s is None or s == "" or len(s) > 19 or (len(s) > 1 and s[0] == "0"): + raise ProtocolError("invalid decimal") + if not all("0" <= c <= "9" for c in s): + raise ProtocolError("invalid decimal") + n = int(s) + if n > MAX_WIRE_INTEGER: + raise ProtocolError("invalid decimal") + return n + + +def validate_etag(s: str) -> None: + if len(s) < 2 or len(s) > MAX_FIELD_BYTES or s[0] != '"' or s[-1] != '"': + raise RacerError("invalid etag") + for c in s[1:-1]: + o = ord(c) + if o != 0x21 and not 0x23 <= o <= 0x7E: + raise RacerError("invalid etag") + + +def validate_opaque(s: str) -> None: + raw = s.encode() + if len(raw) > MAX_FIELD_BYTES: + raise RacerError("header field too large") + if not raw or raw[0] == 0x20 or raw[-1] == 0x20 or any(b < 0x20 or b == 0x7F for b in raw): + raise RacerError("invalid header field") + + +def validate_content_type(s: str) -> bool: + return 0 < len(s) <= 256 and s.strip() == s and all(0x20 <= ord(c) <= 0x7E for c in s) + + +@dataclass +class ResponseHead: + status: int + headers: dict[str, str] # lower-case names + + def get(self, name: str) -> str | None: + return self.headers.get(name.lower()) + + +_SINGLETONS = { + "content-length", + "content-type", + "content-range", + "etag", + "racer-expires-at", + "racer-content-type", + "racer-object-length", + "racer-range-start", + "racer-range-end", + "racer-retain-until", + "racer-delivery", +} + + +def parse_response_head(raw: bytes) -> ResponseHead: + """Parse an HTTP/1.1 response head ending in CRLF CRLF.""" + try: + text = raw.decode("latin-1") + except Exception as exc: # pragma: no cover - latin-1 never fails + raise ProtocolError("bad head") from exc + lines = text.split("\r\n") + if len(lines) < 3 or lines[-1] != "" or lines[-2] != "": + raise ProtocolError("bad head") + line = lines[0] + if len(line) < 12 or not line.startswith("HTTP/1.1 ") or (len(line) > 12 and line[12] != " "): + raise ProtocolError("bad status line") + code = line[9:12] + if not code.isdigit(): + raise ProtocolError("bad status line") + headers: dict[str, str] = {} + for h in lines[1:-2]: + name, sep, value = h.partition(":") + if not sep or not name or name != name.strip(): + raise ProtocolError("bad header") + key = name.lower() + value = value.strip(" \t") + if key in headers: + if key in _SINGLETONS: + raise ProtocolError("duplicate header " + name) + headers[key] += ", " + value + else: + headers[key] = value + return ResponseHead(int(code), headers) + + +def connection_close(h: ResponseHead) -> bool: + v = h.get("connection") or "" + return any(t.strip().lower() == "close" for t in v.split(",")) + + +def check_error_status(h: ResponseHead) -> None: + """Raise the mapped error for a non-200 response, like subscriptionStatusError.""" + if h.status == 200: + return + if decimal(h.get("content-length")) != 0: + raise ProtocolError("error response with a body") + if h.status == 416: + v = h.get("content-range") or "" + if not v.startswith("bytes */"): + raise ProtocolError("416 without Content-Range") + decimal(v[len("bytes */") :]) + elif h.get("content-range") is not None: + raise ProtocolError("unexpected Content-Range") + raise status_error(h.status) diff --git a/clients/python/racer/src/racer/client.py b/clients/python/racer/src/racer/client.py new file mode 100644 index 000000000..e541d2743 --- /dev/null +++ b/clients/python/racer/src/racer/client.py @@ -0,0 +1,680 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Racer client over the node-local Unix socket. + +Request heads and response checks mirror pkg/racersdk (client.go, object.go) +and pkg/racersdk/internal/wire/wire.go. The host-read and gpu-read RPCs follow +designs/racer-gpudirect.md. +""" + +from __future__ import annotations + +import array +import contextlib +import os +import socket +import threading +from dataclasses import dataclass +from typing import Callable, Optional + +from . import _wire as wire +from .errors import ProtocolError, RacerError, StatusError, TruncatedError, UnsatisfiableRange +from .keys import Request + +OnPage = Optional[Callable[[int, int], None]] + +GPU = "gpu" +HOST = "host" +STREAM = "stream" + +# Statuses that mean "this dataplane does not support the mode" on a first +# attempt (designs/racer-gpudirect.md, Fallback). +FALLBACK_STATUSES = frozenset({400, 404, 405, 501}) + +DEFAULT_STAGING_BYTES = 4 * wire.PAGE_SIZE + +_SUPPORTED = "supported" +_UNSUPPORTED = "unsupported" + +# Mode support per socket path, kept until the process exits. +_mode_lock = threading.Lock() +_mode_state: dict[str, dict[str, str]] = {} + + +def socket_path_for(cache: str | None = None, socket_path: str | None = None) -> str: + """Resolve the client socket path. + + Order: explicit socket_path, then $RACER_SOCKET, then + /run/racer//client/socket (pkg/racersdk/client.go). + """ + if socket_path: + return socket_path + env = os.environ.get("RACER_SOCKET") + if env: + return env + if not cache: + raise RacerError("need a cache name, a socket path, or RACER_SOCKET") + if "/" in cache or cache in (".", ".."): + raise RacerError("invalid cache name") + return f"/run/racer/{cache}/client/socket" + + +def forget_modes(path: str | None = None) -> None: + """Clear remembered mode support (for tests, or after a dataplane upgrade).""" + with _mode_lock: + if path is None: + _mode_state.clear() + else: + _mode_state.pop(path, None) + + +@dataclass(frozen=True) +class Metadata: + etag: str + size: int + expires_at_ms: int + content_type: str | None = None + retain_until_ms: int | None = None + + +@dataclass(frozen=True) +class ReadResult: + metadata: Metadata + first: int + end: int + delivery: str + pages: int + + @property + def length(self) -> int: + return self.end - self.first + + +class _Conn: + def __init__(self, path: str, timeout: float | None) -> None: + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.settimeout(timeout) + try: + self.sock.connect(path) + except BaseException: + self.sock.close() + raise + self.buf = bytearray() + + def close(self) -> None: + self.sock.close() + + def send_head(self, head: bytes, fd: int | None = None) -> None: + if fd is None: + self.sock.sendall(head) + return + # The descriptor rides on the sendmsg that carries the first head byte. + anc = [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", [fd]).tobytes())] + sent = self.sock.sendmsg([head], anc) + if sent < len(head): + self.sock.sendall(head[sent:]) + + def send_credit(self, number: int, length: int) -> None: + # Racer may finish and close while a credit is in flight; the next read + # reports whether the transfer was cut short (object.go credit()). + with contextlib.suppress(BrokenPipeError, ConnectionResetError): + self.sock.sendall(wire.encode_credit(number, length)) + + def read_head(self) -> bytes: + while True: + i = self.buf.find(b"\r\n\r\n") + if i >= 0: + head = bytes(self.buf[: i + 4]) + del self.buf[: i + 4] + return head + if len(self.buf) > wire.MAX_HEAD_BYTES: + raise ProtocolError("response head too large") + chunk = self.sock.recv(65536) + if not chunk: + raise TruncatedError("connection closed before the response head") + self.buf += chunk + + def readinto(self, mv: memoryview) -> None: + n = min(len(self.buf), len(mv)) + if n: + mv[:n] = self.buf[:n] + del self.buf[:n] + while n < len(mv): + got = self.sock.recv_into(mv[n:]) + if got == 0: + raise TruncatedError("connection closed mid-transfer") + n += got + + def read_exact(self, size: int) -> bytes: + out = bytearray(size) + self.readinto(memoryview(out)) + return bytes(out) + + +def _response_metadata(h: wire.ResponseHead) -> Metadata | None: + etag = h.get("etag") or "" + try: + wire.validate_etag(etag) + expiry = wire.decimal(h.get("racer-expires-at")) + except RacerError: + return None + ct = h.get("racer-content-type") + if ct is not None and not wire.validate_content_type(ct): + return None + return Metadata(etag=etag, size=0, expires_at_ms=expiry, content_type=ct) + + +class Client: + """Reads Racer objects through the client socket. + + Each read uses its own connection, so one Client may be shared by threads. + """ + + def __init__( + self, + cache: str | None = None, + socket_path: str | None = None, + timeout: float | None = 60.0, + staging_bytes: int = DEFAULT_STAGING_BYTES, + pin_staging: bool = True, + ) -> None: + self.socket_path = socket_path_for(cache, socket_path) + self.timeout = timeout + if staging_bytes <= 0: + raise RacerError("staging_bytes must be positive") + self.staging_bytes = staging_bytes + self.pin_staging = pin_staging + self._pool: list = [] + self._pool_lock = threading.Lock() + + # Mode memory ----------------------------------------------------------- + + def mode_state(self, mode: str) -> str | None: + with _mode_lock: + return _mode_state.get(self.socket_path, {}).get(mode) + + def _set_mode(self, mode: str, state: str) -> None: + with _mode_lock: + _mode_state.setdefault(self.socket_path, {})[mode] = state + + # Staging pool ---------------------------------------------------------- + + def _take_staging(self): + from .host import HostBuffer + + with self._pool_lock: + if self._pool: + return self._pool.pop() + buf = HostBuffer(self.staging_bytes, "racer-staging") + if self.pin_staging: + buf.pin() + return buf + + def _give_staging(self, buf) -> None: + with self._pool_lock: + self._pool.append(buf) + + def close(self) -> None: + with self._pool_lock: + pool, self._pool = self._pool, [] + for b in pool: + b.close() + + def __enter__(self) -> "Client": + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + # Heads ----------------------------------------------------------------- + + @staticmethod + def _context_headers(request: Request) -> str: + out = "" + if request.metadata: + wire.validate_opaque(request.metadata) + out += "Racer-Metadata: " + request.metadata + "\r\n" + if request.authorization: + wire.validate_opaque(request.authorization) + out += "Authorization: " + request.authorization + "\r\n" + return out + + @staticmethod + def _range_header(offset: int, length: int | None) -> str: + if offset < 0 or (length is not None and length < 0) or offset > wire.MAX_WIRE_INTEGER: + raise RacerError("invalid range") + if length and offset + length - 1 > wire.MAX_WIRE_INTEGER: + raise RacerError("invalid range") + if offset == 0 and not length: + return "" + out = f"Range: bytes={offset}-" + if length: + out += str(offset + length - 1) + return out + "\r\n" + + def _head_bytes(self, head: str) -> bytes: + raw = head.encode() + if len(raw) > wire.MAX_HEAD_BYTES: + raise RacerError("request head too large") + return raw + + def _connect(self) -> _Conn: + return _Conn(self.socket_path, self.timeout) + + # Stat ------------------------------------------------------------------ + + def stat(self, request: Request, if_match: str | None = None, retain_seconds: int | None = None) -> Metadata: + """HEAD an object. retain_seconds asks Racer to keep it warm.""" + head = f"HEAD {wire.OBJECT_PREFIX}{request.hex} HTTP/1.1\r\nHost: racer\r\n" + if if_match: + wire.validate_etag(if_match) + head += "If-Match: " + if_match + "\r\n" + if retain_seconds: + head += f"Cache-Control: max-age={int(retain_seconds)}\r\n" + head += self._context_headers(request) + "\r\n" + conn = self._connect() + try: + conn.send_head(self._head_bytes(head)) + h = wire.parse_response_head(conn.read_head()) + finally: + conn.close() + wire.check_error_status(h) + size = wire.decimal(h.get("content-length")) + if h.get("content-type") != "application/octet-stream": + raise ProtocolError("bad Content-Type") + m = _response_metadata(h) + if m is None or (if_match and m.etag != if_match): + raise ProtocolError("bad HEAD metadata") + retain = h.get("racer-retain-until") + return Metadata( + etag=m.etag, + size=size, + expires_at_ms=m.expires_at_ms, + content_type=m.content_type, + retain_until_ms=wire.decimal(retain) if retain is not None else None, + ) + + # Shared response checks ------------------------------------------------ + + @staticmethod + def _check_selection( + h: wire.ResponseHead, offset: int, length: int | None, if_match: str | None + ) -> tuple[Metadata, int, int]: + """Validate a 200 subscription-style head (wire.ParseSubscriptionResponse).""" + if not wire.connection_close(h) or h.get("content-range") is not None: + raise ProtocolError("bad response head") + if h.get("content-type") != "application/octet-stream": + raise ProtocolError("bad Content-Type") + size = wire.decimal(h.get("racer-object-length")) + first = wire.decimal(h.get("racer-range-start")) + end = wire.decimal(h.get("racer-range-end")) + m = _response_metadata(h) + if m is None or first > end or end > size or first != offset: + raise ProtocolError("bad response selection") + if not length and end != size: + raise ProtocolError("bad response selection") + if if_match and m.etag != if_match: + raise ProtocolError("response ETag does not match If-Match") + if length and end - first != length: + if end == size and length > end - first: + raise UnsatisfiableRange(416, "range extends past the object") + raise ProtocolError("bad response selection") + meta = Metadata(m.etag, size, m.expires_at_ms, m.content_type) + return meta, first, end + + # Stream mode ----------------------------------------------------------- + + def _stream( + self, + request: Request, + offset: int, + length: int | None, + if_match: str | None, + scratch: memoryview, + write: Callable[[int, memoryview], None], + on_page: OnPage, + max_length: int | None = None, + on_head: Callable[[], None] | None = None, + ) -> ReadResult: + head = ( + f"POST {wire.OBJECT_PREFIX}{request.hex} HTTP/1.1\r\nHost: racer\r\nContent-Length: 0\r\n" + f"Racer-Page-Credits: {wire.PAGE_CREDITS}\r\nRacer-Byte-Credits: {wire.BYTE_CREDITS}\r\n" + ) + head += self._range_header(offset, length) + if if_match: + wire.validate_etag(if_match) + head += "If-Match: " + if_match + "\r\n" + head += self._context_headers(request) + "\r\n" + if len(scratch) == 0: + raise RacerError("empty scratch buffer") + conn = self._connect() + try: + conn.send_head(self._head_bytes(head)) + h = wire.parse_response_head(conn.read_head()) + wire.check_error_status(h) + if on_head: + on_head() + meta, first, end = self._check_selection(h, offset, length, if_match) + pages = wire.page_count(first, end) + body = wire.decimal(h.get("content-length")) + if body != end - first + wire.FRAME_SIZE * (pages + 1): + raise ProtocolError("bad Content-Length") + if max_length is not None and end - first > max_length: + raise RacerError("destination is smaller than the range") + seq = wire.Sequence(first, end) + for _ in range(pages): + f = wire.decode_frame(conn.read_exact(wire.FRAME_SIZE)) + seq.accept(f) + if f.kind != wire.PAGE_FRAME: + raise ProtocolError("missing page frame") + pos = 0 + while pos < f.length: + n = min(len(scratch), f.length - pos) + conn.readinto(scratch[:n]) + write(f.offset + pos, scratch[:n]) + pos += n + conn.send_credit(f.number, f.length) + if on_page: + on_page(f.offset, f.length) + f = wire.decode_frame(conn.read_exact(wire.FRAME_SIZE)) + seq.accept(f) + if f.kind != wire.COMPLETE_FRAME: # pragma: no cover - accept() enforces it + raise ProtocolError("missing completion") + return ReadResult(meta, first, end, STREAM, pages) + finally: + conn.close() + + def read(self, request: Request, offset: int = 0, length: int | None = None, if_match: str | None = None) -> bytes: + """Read a range into new bytes using stream mode.""" + out = bytearray() + + def write(o: int, piece: memoryview) -> None: + out.extend(piece) + + self._stream(request, offset, length, if_match, memoryview(bytearray(1 << 20)), write, None) + return bytes(out) + + # RPC modes ------------------------------------------------------------- + + def _rpc( + self, + mode: str, + request: Request, + fd: int, + dest_offset: int, + dest_length: int, + allocation_size: int | None, + offset: int, + length: int | None, + if_match: str | None, + on_frame: Callable[[int, int], None] | None, + on_head: Callable[[], None] | None = None, + ) -> ReadResult: + if dest_offset < 0 or dest_length <= 0: + raise RacerError("invalid destination window") + head = f"POST {wire.OBJECT_PREFIX}{request.hex}/{mode}-read HTTP/1.1\r\nHost: racer\r\nContent-Length: 0\r\n" + head += self._range_header(offset, length) + if if_match: + wire.validate_etag(if_match) + head += "If-Match: " + if_match + "\r\n" + head += f"Racer-Destination-Offset: {dest_offset}\r\nRacer-Destination-Length: {dest_length}\r\n" + if allocation_size is not None: + head += f"Racer-Allocation-Size: {allocation_size}\r\n" + head += self._context_headers(request) + "\r\n" + conn = self._connect() + try: + conn.send_head(self._head_bytes(head), fd) + h = wire.parse_response_head(conn.read_head()) + wire.check_error_status(h) + if on_head: + on_head() + if h.get("racer-delivery") != mode: + raise ProtocolError("bad Racer-Delivery") + meta, first, end = self._check_selection(h, offset, length, if_match) + if end - first > dest_length: + raise ProtocolError("range is larger than the destination") + pages = wire.page_count(first, end) + if wire.decimal(h.get("content-length")) != wire.FRAME_SIZE * (pages + 1): + raise ProtocolError("bad Content-Length") + seq = wire.Sequence(first, end) + for _ in range(pages): + f = wire.decode_frame(conn.read_exact(wire.FRAME_SIZE)) + seq.accept(f) + if f.kind != wire.PAGE_FRAME: + raise ProtocolError("missing page frame") + if on_frame: + on_frame(f.offset, f.length) + f = wire.decode_frame(conn.read_exact(wire.FRAME_SIZE)) + seq.accept(f) + return ReadResult(meta, first, end, mode, pages) + finally: + conn.close() + + def host_read( + self, + request: Request, + fd: int, + dest_offset: int, + dest_length: int, + *, + offset: int = 0, + length: int | None = None, + if_match: str | None = None, + on_page: OnPage = None, + ) -> ReadResult: + """One host-read RPC with no fallback. fd is a regular file, normally a memfd.""" + return self._rpc(HOST, request, fd, dest_offset, dest_length, None, offset, length, if_match, on_page) + + def gpu_read( + self, + request: Request, + fd: int, + allocation_size: int, + dest_offset: int, + dest_length: int, + *, + offset: int = 0, + length: int | None = None, + if_match: str | None = None, + on_page: OnPage = None, + ) -> ReadResult: + """One gpu-read RPC with no fallback. fd comes from cuMemExportToShareableHandle.""" + if dest_offset + dest_length > allocation_size: + raise RacerError("destination window is outside the allocation") + return self._rpc(GPU, request, fd, dest_offset, dest_length, allocation_size, offset, length, if_match, on_page) + + # Fallback driver ------------------------------------------------------- + + def _run_modes(self, attempts: list[tuple[str, Callable[[Callable[[], None]], ReadResult]]]) -> ReadResult: + """Try modes in order. + + A mode with no remembered state that answers 400/404/405/501 is + skipped. 501 marks it unsupported at once. 400/404/405 mark it + unsupported only once a later mode gets a 200 head, so a missing + object (404 everywhere) does not disable a mode for good. + """ + pending: list[str] = [] + last: StatusError | None = None + + def understood() -> None: + for m in pending: + self._set_mode(m, _UNSUPPORTED) + pending.clear() + + for mode, run in attempts: + state = self.mode_state(mode) if mode != STREAM else _SUPPORTED + if state == _UNSUPPORTED: + continue + try: + result = run(understood) + except _Skip: + continue + except StatusError as exc: + if mode == STREAM or state is not None or exc.status not in FALLBACK_STATUSES: + raise + if exc.status == 501: + self._set_mode(mode, _UNSUPPORTED) + else: + pending.append(mode) + last = exc + continue + if mode != STREAM: + self._set_mode(mode, _SUPPORTED) + return result + if last is not None: # pragma: no cover - stream always runs last + raise last + raise RacerError("no read mode available") + + # High level: host ------------------------------------------------------ + + def read_into_host( + self, + request: Request, + dest, + dest_offset: int = 0, + *, + offset: int = 0, + length: int | None = None, + if_match: str | None = None, + on_page: OnPage = None, + ) -> ReadResult: + """Read a range into host memory at dest_offset. + + dest is a HostBuffer (host-read, then stream) or any writable buffer + (stream only). Object offset o lands at dest_offset + (o - offset). + """ + view = memoryview(dest.view if hasattr(dest, "view") else dest).cast("B") + window = len(view) - dest_offset + if dest_offset < 0 or window <= 0: + raise RacerError("invalid destination window") + if length and length > window: + raise RacerError("destination is smaller than the range") + attempts = [] + if hasattr(dest, "fileno"): + attempts.append( + ( + HOST, + lambda ok: self._rpc( + HOST, request, dest.fileno(), dest_offset, window, None, offset, length, if_match, on_page, ok + ), + ) + ) + + def stream(ok): + def write(o: int, piece: memoryview) -> None: + at = dest_offset + o - offset + view[at : at + len(piece)] = piece + + scratch = memoryview(bytearray(min(window, 1 << 20))) + return self._stream(request, offset, length, if_match, scratch, write, on_page, window, ok) + + attempts.append((STREAM, stream)) + try: + return self._run_modes(attempts) + finally: + view.release() + + # High level: GPU ------------------------------------------------------- + + def read_into_gpu( + self, + request: Request, + dest, + dest_offset: int = 0, + *, + offset: int = 0, + length: int | None = None, + if_match: str | None = None, + on_page: OnPage = None, + ) -> ReadResult: + """Read a range into GPU memory at dest_offset. + + dest is a racer.cuda.GpuAllocation, a racer.tensors.TensorTarget, or + any object with ``size``, ``copy_from_host(offset, data)`` and + optionally ``export_fd()``. Modes: gpu-read, then host-read into a + pinned staging memfd plus a host-to-device copy, then stream into the + staging buffer plus a copy. on_page(object_offset, length) runs once a + page is in GPU memory. + """ + window = dest.size - dest_offset + if dest_offset < 0 or window <= 0: + raise RacerError("invalid destination window") + if length and length > window: + raise RacerError("destination is smaller than the range") + + attempts: list = [] + if hasattr(dest, "export_fd"): + + def gpu(ok): + try: + fd = dest.export_fd() + except NotImplementedError: + raise _Skip() from None + try: + return self._rpc( + GPU, request, fd, dest_offset, window, dest.size, offset, length, if_match, on_page, ok + ) + finally: + os.close(fd) + + attempts.append((GPU, gpu)) + attempts.append((HOST, lambda ok: self._gpu_via_host(request, dest, dest_offset, window, offset, length, if_match, on_page, ok))) + attempts.append((STREAM, lambda ok: self._gpu_via_stream(request, dest, dest_offset, window, offset, length, if_match, on_page, ok))) + + return self._run_modes(attempts) + + def _gpu_via_host(self, request, dest, dest_offset, window, offset, length, if_match, on_page, ok) -> ReadResult: + staging = self._take_staging() + try: + chunk = staging.size + if length: + total = length + else: + # Resolve the size once so every chunk reads the same version. + meta = self.stat(request, if_match=if_match) + if offset > meta.size: + raise UnsatisfiableRange(416) + total = meta.size - offset + if_match = meta.etag + ok() + if total > window: + raise RacerError("destination is smaller than the range") + if total == 0: + return self._rpc(HOST, request, staging.fileno(), 0, chunk, None, offset, None, if_match, None, ok) + first_result = None + pos = 0 + pages = 0 + while pos < total: + n = min(chunk, total - pos) + base = offset + pos + + def frame(o: int, size: int, base=base) -> None: + dest.copy_from_host(dest_offset + (o - offset), staging.view[o - base : o - base + size]) + if on_page: + on_page(o, size) + + r = self._rpc(HOST, request, staging.fileno(), 0, n, None, base, n, if_match, frame, ok) + if first_result is None: + first_result = r + if_match = r.metadata.etag + pages += r.pages + pos += n + assert first_result is not None + return ReadResult(first_result.metadata, offset, offset + total, HOST, pages) + finally: + self._give_staging(staging) + + def _gpu_via_stream(self, request, dest, dest_offset, window, offset, length, if_match, on_page, ok) -> ReadResult: + staging = self._take_staging() + try: + + def write(o: int, piece: memoryview) -> None: + dest.copy_from_host(dest_offset + (o - offset), piece) + + return self._stream(request, offset, length, if_match, staging.view, write, on_page, window, ok) + finally: + self._give_staging(staging) + + +class _Skip(Exception): + """The destination cannot use this mode. Nothing is remembered.""" diff --git a/clients/python/racer/src/racer/cuda.py b/clients/python/racer/src/racer/cuda.py new file mode 100644 index 000000000..642e56963 --- /dev/null +++ b/clients/python/racer/src/racer/cuda.py @@ -0,0 +1,377 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Minimal ctypes bindings to the CUDA driver API (libcuda.so.1). + +Only what Racer needs: exportable device allocations through the virtual +memory management (VMM) API, host-to-device copies, and host registration. + +Struct layouts and enum values come from cuda.h in the CUDA toolkit +(https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__TYPES.html and +https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__VA.html): + + typedef struct CUmemLocation_st { CUmemLocationType type; int id; } CUmemLocation; + typedef struct CUmemAllocationProp_st { + CUmemAllocationType type; + CUmemAllocationHandleType requestedHandleTypes; + CUmemLocation location; + void *win32HandleMetaData; + struct { + unsigned char compressionType; + unsigned char gpuDirectRDMACapable; + unsigned short usage; + unsigned char reserved[4]; + } allocFlags; + } CUmemAllocationProp; + typedef struct CUmemAccessDesc_st { CUmemLocation location; CUmemAccess_flags flags; } CUmemAccessDesc; + + CU_MEM_ALLOCATION_TYPE_PINNED = 0x1 + CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR = 0x1 + CU_MEM_LOCATION_TYPE_DEVICE = 0x1 + CU_MEM_ACCESS_FLAGS_PROT_READWRITE = 0x3 + CU_MEM_ALLOC_GRANULARITY_MINIMUM = 0x0 + +Versioned symbols follow the cuda.h #defines: cuMemcpyHtoD -> cuMemcpyHtoD_v2, +cuMemHostRegister -> cuMemHostRegister_v2, cuDevicePrimaryCtxRelease -> +cuDevicePrimaryCtxRelease_v2. +""" + +from __future__ import annotations + +import ctypes +import logging +import os +import threading + +from .errors import RacerError + +log = logging.getLogger(__name__) + +CU_MEM_ALLOCATION_TYPE_PINNED = 0x1 +CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR = 0x1 +CU_MEM_LOCATION_TYPE_DEVICE = 0x1 +CU_MEM_ACCESS_FLAGS_PROT_READWRITE = 0x3 +CU_MEM_ALLOC_GRANULARITY_MINIMUM = 0x0 +CU_MEMHOSTREGISTER_PORTABLE = 0x1 + +CUdeviceptr = ctypes.c_ulonglong +CUmemGenericAllocationHandle = ctypes.c_ulonglong + + +class CudaUnavailable(RacerError): + """libcuda is missing or the driver cannot be initialized.""" + + +class CudaError(RacerError): + def __init__(self, func: str, code: int) -> None: + super().__init__(f"{func} failed with CUresult {code}") + self.func = func + self.code = code + + +class CUmemLocation(ctypes.Structure): + _fields_ = [("type", ctypes.c_int), ("id", ctypes.c_int)] + + +class _AllocFlags(ctypes.Structure): + _fields_ = [ + ("compressionType", ctypes.c_ubyte), + ("gpuDirectRDMACapable", ctypes.c_ubyte), + ("usage", ctypes.c_ushort), + ("reserved", ctypes.c_ubyte * 4), + ] + + +class CUmemAllocationProp(ctypes.Structure): + _fields_ = [ + ("type", ctypes.c_int), + ("requestedHandleTypes", ctypes.c_int), + ("location", CUmemLocation), + ("win32HandleMetaData", ctypes.c_void_p), + ("allocFlags", _AllocFlags), + ] + + +class CUmemAccessDesc(ctypes.Structure): + _fields_ = [("location", CUmemLocation), ("flags", ctypes.c_int)] + + +_lock = threading.Lock() +_lib = None +_contexts: dict[int, ctypes.c_void_p] = {} + + +def _load(): + global _lib + with _lock: + if _lib is not None: + return _lib + name = os.environ.get("RACER_LIBCUDA", "libcuda.so.1") + try: + lib = ctypes.CDLL(name) + except OSError as exc: + raise CudaUnavailable(f"cannot load {name}: {exc}") from exc + sig = { + "cuInit": [ctypes.c_uint], + "cuDeviceGet": [ctypes.POINTER(ctypes.c_int), ctypes.c_int], + "cuCtxGetCurrent": [ctypes.POINTER(ctypes.c_void_p)], + "cuCtxSetCurrent": [ctypes.c_void_p], + "cuDevicePrimaryCtxRetain": [ctypes.POINTER(ctypes.c_void_p), ctypes.c_int], + "cuMemGetAllocationGranularity": [ + ctypes.POINTER(ctypes.c_size_t), + ctypes.POINTER(CUmemAllocationProp), + ctypes.c_int, + ], + "cuMemCreate": [ + ctypes.POINTER(CUmemGenericAllocationHandle), + ctypes.c_size_t, + ctypes.POINTER(CUmemAllocationProp), + ctypes.c_ulonglong, + ], + "cuMemRelease": [CUmemGenericAllocationHandle], + "cuMemAddressReserve": [ + ctypes.POINTER(CUdeviceptr), + ctypes.c_size_t, + ctypes.c_size_t, + CUdeviceptr, + ctypes.c_ulonglong, + ], + "cuMemAddressFree": [CUdeviceptr, ctypes.c_size_t], + "cuMemMap": [CUdeviceptr, ctypes.c_size_t, ctypes.c_size_t, CUmemGenericAllocationHandle, ctypes.c_ulonglong], + "cuMemUnmap": [CUdeviceptr, ctypes.c_size_t], + "cuMemSetAccess": [CUdeviceptr, ctypes.c_size_t, ctypes.POINTER(CUmemAccessDesc), ctypes.c_size_t], + "cuMemExportToShareableHandle": [ + ctypes.c_void_p, + CUmemGenericAllocationHandle, + ctypes.c_int, + ctypes.c_ulonglong, + ], + "cuMemcpyHtoD_v2": [CUdeviceptr, ctypes.c_void_p, ctypes.c_size_t], + "cuMemcpyDtoH_v2": [ctypes.c_void_p, CUdeviceptr, ctypes.c_size_t], + "cuMemHostRegister_v2": [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint], + "cuMemHostUnregister": [ctypes.c_void_p], + } + for fn, args in sig.items(): + f = getattr(lib, fn, None) + if f is None: + raise CudaUnavailable(f"libcuda has no {fn}") + f.argtypes = args + f.restype = ctypes.c_int + rc = lib.cuInit(0) + if rc != 0: + raise CudaUnavailable(f"cuInit failed with CUresult {rc}") + _lib = lib + return lib + + +def _check(func: str, rc: int) -> None: + if rc != 0: + raise CudaError(func, rc) + + +def available() -> bool: + try: + _load() + except CudaUnavailable: + return False + return True + + +def ensure_context(device: int = 0) -> None: + """Make sure a context is current; retain the primary context if not. + + torch uses the primary context too, so both share one context. + """ + lib = _load() + cur = ctypes.c_void_p() + _check("cuCtxGetCurrent", lib.cuCtxGetCurrent(ctypes.byref(cur))) + if cur.value: + return + with _lock: + ctx = _contexts.get(device) + if ctx is None: + dev = ctypes.c_int() + _check("cuDeviceGet", lib.cuDeviceGet(ctypes.byref(dev), device)) + ctx = ctypes.c_void_p() + _check("cuDevicePrimaryCtxRetain", lib.cuDevicePrimaryCtxRetain(ctypes.byref(ctx), dev.value)) + _contexts[device] = ctx + _check("cuCtxSetCurrent", lib.cuCtxSetCurrent(ctx)) + + +def _address(buf) -> tuple[int, object]: + """Return the address of a buffer and an object that keeps it valid.""" + mv = memoryview(buf).cast("B") + if mv.readonly: + holder = ctypes.create_string_buffer(mv.tobytes(), len(mv)) + return ctypes.addressof(holder), holder + arr = (ctypes.c_char * len(mv)).from_buffer(mv) + return ctypes.addressof(arr), arr + + +def _allocation_prop(device: int) -> CUmemAllocationProp: + prop = CUmemAllocationProp() + prop.type = CU_MEM_ALLOCATION_TYPE_PINNED + prop.requestedHandleTypes = CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR + prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE + prop.location.id = device + return prop + + +def granularity(device: int = 0) -> int: + lib = _load() + ensure_context(device) + g = ctypes.c_size_t() + prop = _allocation_prop(device) + _check( + "cuMemGetAllocationGranularity", + lib.cuMemGetAllocationGranularity(ctypes.byref(g), ctypes.byref(prop), CU_MEM_ALLOC_GRANULARITY_MINIMUM), + ) + return g.value + + +class GpuAllocation: + """Exportable device memory, mapped read-write in this process. + + ``size`` is the size passed to cuMemCreate, rounded up to the allocation + granularity. It is what gpu-read sends as Racer-Allocation-Size. + + The object implements ``__cuda_array_interface__`` (a flat uint8 array), + so torch, CuPy and Numba can wrap it without a copy. Any wrapper must keep + this object alive; ``racer.torch.as_tensor`` does. + """ + + def __init__(self, size: int, device: int = 0) -> None: + if size <= 0: + raise RacerError("size must be positive") + lib = _load() + ensure_context(device) + g = granularity(device) + self.device = device + self.requested = size + self.size = (size + g - 1) // g * g + self.ptr = 0 + self._handle = None + self._reserved = False + self._mapped = False + prop = _allocation_prop(device) + handle = CUmemGenericAllocationHandle() + try: + _check("cuMemCreate", lib.cuMemCreate(ctypes.byref(handle), self.size, ctypes.byref(prop), 0)) + self._handle = handle.value + ptr = CUdeviceptr() + _check("cuMemAddressReserve", lib.cuMemAddressReserve(ctypes.byref(ptr), self.size, g, 0, 0)) + self.ptr = ptr.value + self._reserved = True + _check("cuMemMap", lib.cuMemMap(self.ptr, self.size, 0, self._handle, 0)) + self._mapped = True + desc = CUmemAccessDesc() + desc.location.type = CU_MEM_LOCATION_TYPE_DEVICE + desc.location.id = device + desc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE + _check("cuMemSetAccess", lib.cuMemSetAccess(self.ptr, self.size, ctypes.byref(desc), 1)) + except BaseException: + self.close() + raise + + def export_fd(self) -> int: + """Return a new POSIX file descriptor for the allocation. The caller closes it.""" + lib = _load() + fd = ctypes.c_int(-1) + _check( + "cuMemExportToShareableHandle", + lib.cuMemExportToShareableHandle(ctypes.byref(fd), self._handle, CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR, 0), + ) + return fd.value + + def copy_from_host(self, offset: int, data) -> None: + mv = memoryview(data).cast("B") + if offset < 0 or offset + len(mv) > self.size: + raise RacerError("copy outside the allocation") + if not mv: + return + addr, keep = _address(mv) + try: + _check("cuMemcpyHtoD_v2", _load().cuMemcpyHtoD_v2(self.ptr + offset, addr, len(mv))) + finally: + del keep + + def copy_to_host(self, offset: int, out) -> None: + mv = memoryview(out).cast("B") + if offset < 0 or offset + len(mv) > self.size: + raise RacerError("copy outside the allocation") + addr, keep = _address(mv) + try: + _check("cuMemcpyDtoH_v2", _load().cuMemcpyDtoH_v2(addr, self.ptr + offset, len(mv))) + finally: + del keep + + @property + def __cuda_array_interface__(self) -> dict: + return { + "shape": (self.size,), + "typestr": "|u1", + "data": (self.ptr, False), + "strides": None, + "version": 3, + } + + def close(self) -> None: + """Unmap and release. On failure the resource is kept (leaked), never freed early.""" + if _lib is None: + return + lib = _lib + try: + if self._mapped: + _check("cuMemUnmap", lib.cuMemUnmap(self.ptr, self.size)) + self._mapped = False + if self._reserved: + _check("cuMemAddressFree", lib.cuMemAddressFree(self.ptr, self.size)) + self._reserved = False + if self._handle is not None: + _check("cuMemRelease", lib.cuMemRelease(self._handle)) + self._handle = None + except CudaError as exc: + log.warning("racer: leaking GPU allocation: %s", exc) + self._mapped = self._reserved = False + self._handle = None + + def __enter__(self) -> "GpuAllocation": + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + def __del__(self) -> None: + try: + self.close() + except Exception: + pass + + +class HostRegistration: + def __init__(self, addr: int, keep: object) -> None: + self._addr = addr + self._keep = keep + + def close(self) -> None: + if self._addr: + rc = _load().cuMemHostUnregister(self._addr) + if rc != 0: + log.warning("racer: cuMemHostUnregister failed with CUresult %d", rc) + self._addr = 0 + self._keep = None + + +def host_register(buf) -> HostRegistration: + """Page-lock a writable host buffer with cuMemHostRegister.""" + lib = _load() + ensure_context(0) + mv = memoryview(buf).cast("B") + if mv.readonly: + raise RacerError("buffer must be writable") + arr = (ctypes.c_char * len(mv)).from_buffer(mv) + addr = ctypes.addressof(arr) + _check("cuMemHostRegister_v2", lib.cuMemHostRegister_v2(addr, len(mv), CU_MEMHOSTREGISTER_PORTABLE)) + # Drop the export so the owner can still close its mapping after unregistering. + del arr + mv.release() + return HostRegistration(addr, None) diff --git a/clients/python/racer/src/racer/errors.py b/clients/python/racer/src/racer/errors.py new file mode 100644 index 000000000..f567250f0 --- /dev/null +++ b/clients/python/racer/src/racer/errors.py @@ -0,0 +1,84 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Errors raised by the Racer client. + +Status mapping follows wire.StatusError in pkg/racersdk/internal/wire/wire.go. +""" + +from __future__ import annotations + + +class RacerError(Exception): + """Base class for all Racer client errors.""" + + +class ProtocolError(RacerError): + """The server sent something the protocol does not allow.""" + + +class TruncatedError(RacerError): + """The connection closed before the completion frame.""" + + +class StatusError(RacerError): + """The server answered with a non-200 status.""" + + status = 0 + + def __init__(self, status: int, message: str | None = None) -> None: + super().__init__(message or f"racer: status {status}") + self.status = status + + +class InvalidArgument(StatusError): + pass + + +class Unauthorized(StatusError): + pass + + +class Forbidden(StatusError): + pass + + +class NotFound(StatusError): + pass + + +class VersionUnavailable(StatusError): + pass + + +class UnsatisfiableRange(StatusError): + pass + + +class HeaderLimit(StatusError): + pass + + +class NotImplementedStatus(StatusError): + pass + + +class Unavailable(StatusError): + pass + + +_BY_STATUS = { + 400: InvalidArgument, + 401: Unauthorized, + 403: Forbidden, + 404: NotFound, + 405: InvalidArgument, + 412: VersionUnavailable, + 416: UnsatisfiableRange, + 431: HeaderLimit, + 501: NotImplementedStatus, + 503: Unavailable, +} + + +def status_error(status: int) -> StatusError: + return _BY_STATUS.get(status, StatusError)(status) diff --git a/clients/python/racer/src/racer/host.py b/clients/python/racer/src/racer/host.py new file mode 100644 index 000000000..1621ae2d2 --- /dev/null +++ b/clients/python/racer/src/racer/host.py @@ -0,0 +1,71 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Host memory backed by a memfd, the destination for host-read.""" + +from __future__ import annotations + +import mmap +import os + +from .errors import RacerError + + +class HostBuffer: + """A memfd of a fixed size, mapped read-write into this process. + + The dataplane writes into the file with positional writes, so the memfd is + not sealed. ``view`` is a writable memoryview over the mapping, so the + buffer also works as a stream-mode destination. Call ``pin()`` to register + the mapping with CUDA for faster host-to-device copies. + """ + + def __init__(self, size: int, name: str = "racer-host") -> None: + if size <= 0: + raise RacerError("size must be positive") + self.size = size + self._fd = os.memfd_create(name, os.MFD_CLOEXEC) + try: + os.ftruncate(self._fd, size) + self._map = mmap.mmap(self._fd, size, mmap.MAP_SHARED, mmap.PROT_READ | mmap.PROT_WRITE) + except BaseException: + os.close(self._fd) + raise + self.view = memoryview(self._map) + self._pinned = None + + def fileno(self) -> int: + if self._fd < 0: + raise RacerError("buffer closed") + return self._fd + + @property + def pinned(self) -> bool: + return self._pinned is not None + + def pin(self) -> bool: + """Register the mapping with CUDA. Returns False when CUDA is missing.""" + if self._pinned is not None: + return True + from . import cuda + + try: + self._pinned = cuda.host_register(self.view) + except cuda.CudaUnavailable: + return False + return True + + def close(self) -> None: + if self._pinned is not None: + self._pinned.close() + self._pinned = None + if self._fd >= 0: + self.view.release() + self._map.close() + os.close(self._fd) + self._fd = -1 + + def __enter__(self) -> "HostBuffer": + return self + + def __exit__(self, *exc: object) -> None: + self.close() diff --git a/clients/python/racer/src/racer/keys.py b/clients/python/racer/src/racer/keys.py new file mode 100644 index 000000000..727ac48cb --- /dev/null +++ b/clients/python/racer/src/racer/keys.py @@ -0,0 +1,88 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Object keys for racer-object (S3) objects. + +Replicates NewRequest in internal/racerobject/racerobject.go: the key is the +SHA-256 of the canonical schema-1 JSON produced by Go's encoding/json, and +that JSON is sent as Racer-Metadata. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass + +from .errors import RacerError + +_BUCKET = re.compile(r"[A-Za-z0-9._-]{1,255}") + + +@dataclass(frozen=True) +class Request: + """One object to read: its 32-byte key plus opaque origin context.""" + + key: bytes + metadata: str | None = None + authorization: str | None = None + + def __post_init__(self) -> None: + if len(self.key) != 32: + raise RacerError("key must be 32 bytes") + + @property + def hex(self) -> str: + return self.key.hex() + + @classmethod + def from_hex(cls, key: str, metadata: str | None = None, authorization: str | None = None) -> "Request": + if len(key) != 64 or any(c not in "0123456789abcdef" for c in key): + raise RacerError("key must be 64 lower-case hex digits") + return cls(bytes.fromhex(key), metadata, authorization) + + +def _go_json_string(s: str) -> str: + # Go escapes <, >, & and U+2028, U+2029 by default (json.Marshal HTMLEscape), + # and writes other non-ASCII as raw UTF-8. Control characters match Python's + # short escapes (\b \f \n \r \t) and \u00XX otherwise. + out = json.dumps(s, ensure_ascii=False) + return ( + out.replace("<", "\\u003c") + .replace(">", "\\u003e") + .replace("&", "\\u0026") + .replace("\u2028", "\\u2028") + .replace("\u2029", "\\u2029") + ) + + +def canonical_json(bucket: str, key: str, version_id: str | None = None) -> str: + """Return the canonical schema-1 JSON, validating like validObject.""" + version_id = version_id or "" + if bucket in (".", "..") or not _BUCKET.fullmatch(bucket): + raise RacerError("invalid bucket") + for s in (key, version_id): + try: + s.encode("utf-8") + except UnicodeEncodeError as exc: + raise RacerError("invalid UTF-8") from exc + if not key or len(key.encode()) > 1024 or len(version_id.encode()) > 1024: + raise RacerError("invalid key or version") + out = '{"schema":1,"bucket":' + _go_json_string(bucket) + ',"key":' + _go_json_string(key) + if version_id: + out += ',"versionId":' + _go_json_string(version_id) + out += "}" + if len(out.encode()) > 8 << 10: + raise RacerError("object identity too large") + return out + + +def object_request(bucket: str, key: str, version_id: str | None = None) -> Request: + """Build the Request racer-object uses for an S3 object.""" + canonical = canonical_json(bucket, key, version_id) + return Request(hashlib.sha256(canonical.encode()).digest(), canonical) + + +def object_key(bucket: str, key: str, version_id: str | None = None) -> str: + """Return the 64-hex-digit Racer key for an S3 object.""" + return object_request(bucket, key, version_id).hex diff --git a/clients/python/racer/src/racer/tensors.py b/clients/python/racer/src/racer/tensors.py new file mode 100644 index 000000000..79418cb13 --- /dev/null +++ b/clients/python/racer/src/racer/tensors.py @@ -0,0 +1,55 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""torch helpers. torch is imported only when these functions run. + +Lifetime rule: a tensor made by ``as_tensor`` holds a reference to the +GpuAllocation it views, so the allocation lives at least as long as the +tensor (and any view of it). Closing the allocation by hand while tensors +still use it is a bug. +""" + +from __future__ import annotations + + +def _torch(): + import torch # noqa: PLC0415 - lazy on purpose + + return torch + + +def as_tensor(allocation, device: int | None = None): + """Wrap a GpuAllocation as a flat uint8 CUDA tensor without copying. + + This uses ``torch.as_tensor`` on the object's ``__cuda_array_interface__``. + torch keeps a reference to the exporting object for as long as the + tensor's storage lives, which gives the lifetime rule above. + """ + torch = _torch() + dev = torch.device("cuda", allocation.device if device is None else device) + return torch.as_tensor(allocation, device=dev) + + +class TensorTarget: + """A GPU destination backed by an existing torch CUDA uint8 tensor. + + It cannot be exported, so reads use host or stream mode and copy with + ``Tensor.copy_``. + """ + + def __init__(self, tensor) -> None: + torch = _torch() + if tensor.dtype != torch.uint8 or tensor.dim() != 1 or not tensor.is_contiguous(): + raise ValueError("tensor must be a contiguous 1-D uint8 tensor") + self.tensor = tensor + self.size = tensor.numel() + + def export_fd(self) -> int: + raise NotImplementedError("torch memory cannot be exported") + + def copy_from_host(self, offset: int, data) -> None: + torch = _torch() + mv = memoryview(data).cast("B") + if not mv: + return + src = torch.frombuffer(mv, dtype=torch.uint8) + self.tensor[offset : offset + len(mv)].copy_(src) diff --git a/clients/python/racer/tests/conftest.py b/clients/python/racer/tests/conftest.py new file mode 100644 index 000000000..30c59c840 --- /dev/null +++ b/clients/python/racer/tests/conftest.py @@ -0,0 +1,67 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +import os +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(__file__)) + +from fakeserver import FakeRacer # noqa: E402 + +import racer # noqa: E402 +from racer import _wire # noqa: E402 + +SMALL_PAGE = 4096 + + +@pytest.fixture +def small_pages(monkeypatch): + monkeypatch.setattr(_wire, "PAGE_SIZE", SMALL_PAGE) + return SMALL_PAGE + + +@pytest.fixture +def server(tmp_path, small_pages): + s = FakeRacer(str(tmp_path / "s")) + racer.forget_modes() + yield s + s.close() + racer.forget_modes() + + +class FakeGpu: + """A GPU destination backed by a memfd, so the fake server can write it.""" + + def __init__(self, size, exportable=True): + self.size = size + self.device = 0 + self.fd = os.memfd_create("fake-gpu") + os.ftruncate(self.fd, size) + self.copies = 0 + if not exportable: + self.export_fd = self._no_export + + def _no_export(self): + raise NotImplementedError + + def export_fd(self): + return os.dup(self.fd) + + def copy_from_host(self, offset, data): + self.copies += 1 + os.pwrite(self.fd, bytes(data), offset) + + def read(self, offset, n): + return os.pread(self.fd, n, offset) + + def close(self): + os.close(self.fd) + + +@pytest.fixture +def gpu_writer(): + def write(fd, offset, data): + os.pwrite(fd, data, offset) + + return write diff --git a/clients/python/racer/tests/fakeserver.py b/clients/python/racer/tests/fakeserver.py new file mode 100644 index 000000000..ec4b3dc4a --- /dev/null +++ b/clients/python/racer/tests/fakeserver.py @@ -0,0 +1,249 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""A small fake Racer client-socket server for tests.""" + +from __future__ import annotations + +import array +import os +import socket +import stat +import threading +from dataclasses import dataclass, field + +from racer import _wire as wire + + +@dataclass +class Obj: + data: bytes + etag: str = '"v1"' + content_type: str | None = None + expires_ms: int = 1_700_000_000_000 + + +@dataclass +class Seen: + method: str + target: str + headers: dict + fds: int + fd_kind: str | None = None + credits: list = field(default_factory=list) + + +class FakeRacer: + """Serves HEAD, stream Get, host-read and gpu-read on a Unix socket. + + old=True mimics a dataplane that predates host-read/gpu-read: the new + paths fail key parsing and get 400. gpu_status is the answer to gpu-read + (501 by default, the "no usable GPU" case). gpu_write, when set, is + called as gpu_write(fd, dest_offset, bytes) to emulate the GPU write. + """ + + def __init__(self, path: str) -> None: + self.path = path + self.objects: dict[str, Obj] = {} + self.old = False + self.gpu_status = 501 + self.gpu_write = None + self.host_status: int | None = None + self.bad_frame = False + self.truncate = False + self.seen: list[Seen] = [] + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.bind(path) + self.sock.listen(16) + self._stop = False + self._thread = threading.Thread(target=self._serve, daemon=True) + self._thread.start() + + def close(self) -> None: + self._stop = True + try: + c = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + c.connect(self.path) + c.close() + except OSError: + pass + self._thread.join(5) + self.sock.close() + + def _serve(self) -> None: + while not self._stop: + conn, _ = self.sock.accept() + if self._stop: + conn.close() + return + threading.Thread(target=self._handle, args=(conn,), daemon=True).start() + + # ------------------------------------------------------------------ + + def _recv_head(self, conn): + buf = b"" + fds: list[int] = [] + while b"\r\n\r\n" not in buf: + data, anc, flags, _ = conn.recvmsg(65536, socket.CMSG_SPACE(4 * 8), socket.MSG_CMSG_CLOEXEC) + for level, typ, cdata in anc: + if level == socket.SOL_SOCKET and typ == socket.SCM_RIGHTS: + a = array.array("i") + a.frombytes(cdata[: len(cdata) - len(cdata) % 4]) + fds.extend(a) + if not data: + return None, None, fds + buf += data + head, _, rest = buf.partition(b"\r\n\r\n") + return head.decode("utf-8", "surrogateescape"), rest, fds + + @staticmethod + def _send(conn, status, headers, body=b""): + lines = [f"HTTP/1.1 {status} X"] + [f"{k}: {v}" for k, v in headers] + conn.sendall(("\r\n".join(lines) + "\r\n\r\n").encode() + body) + + def _error(self, conn, status, size=None): + h = [("Content-Length", "0")] + if status == 416: + h.append(("Content-Range", f"bytes */{size}")) + self._send(conn, status, h) + + def _handle(self, conn) -> None: + fds: list[int] = [] + try: + head, rest, fds = self._recv_head(conn) + if head is None: + return + lines = head.split("\r\n") + method, target, _ = lines[0].split(" ") + headers = {} + for line in lines[1:]: + k, _, v = line.partition(":") + headers[k.strip().lower()] = v.strip() + kind = None + if len(fds) == 1: + kind = "regular" if stat.S_ISREG(os.fstat(fds[0]).st_mode) else "other" + seen = Seen(method, target, headers, len(fds), kind) + self.seen.append(seen) + self._dispatch(conn, method, target, headers, fds, seen, rest) + except OSError: + pass # the client went away, as tests for bad frames expect + finally: + for fd in fds: + os.close(fd) + conn.close() + + def _dispatch(self, conn, method, target, headers, fds, seen, rest): + if headers.get("host") != "racer" or not target.startswith(wire.OBJECT_PREFIX): + return self._error(conn, 400) + tail = target[len(wire.OBJECT_PREFIX) :] + key, _, mode = tail.partition("/") + if mode and (self.old or mode not in ("host-read", "gpu-read")): + return self._error(conn, 400) + if mode and method != "POST": + return self._error(conn, 405) + if fds and not mode: + return self._error(conn, 400) + if mode and len(fds) != 1: + return self._error(conn, 400) + if mode == "gpu-read" and self.gpu_status != 200: + return self._error(conn, self.gpu_status) + if mode == "host-read" and self.host_status: + return self._error(conn, self.host_status) + if mode == "host-read" and seen.fd_kind != "regular": + return self._error(conn, 400) + obj = self.objects.get(key) + if obj is None: + return self._error(conn, 404) + if "if-match" in headers and headers["if-match"] != obj.etag: + return self._error(conn, 412) + meta = [("ETag", obj.etag), ("Racer-Expires-At", str(obj.expires_ms))] + if obj.content_type: + meta.append(("Racer-Content-Type", obj.content_type)) + size = len(obj.data) + if method == "HEAD": + return self._send( + conn, 200, meta + [("Content-Length", str(size)), ("Content-Type", "application/octet-stream")] + ) + if method != "POST": + return self._error(conn, 405) + first, end = 0, size + if "range" in headers: + spec = headers["range"][len("bytes=") :] + a, _, b = spec.partition("-") + first = int(a) + end = min(size, int(b) + 1) if b else size + if first >= size and size > 0 or first > size: + return self._error(conn, 416, size) + pages = wire.page_count(first, end) + sel = meta + [ + ("Content-Type", "application/octet-stream"), + ("Racer-Object-Length", str(size)), + ("Racer-Range-Start", str(first)), + ("Racer-Range-End", str(end)), + ("Connection", "close"), + ] + if mode: + off = int(headers["racer-destination-offset"]) + length = int(headers["racer-destination-length"]) + if length < end - first: + return self._error(conn, 400) + if mode == "gpu-read" and off + length > int(headers["racer-allocation-size"]): + return self._error(conn, 400) + self._send( + conn, + 200, + sel + + [ + ("Racer-Delivery", "host" if mode == "host-read" else "gpu"), + ("Content-Length", str(wire.FRAME_SIZE * (pages + 1))), + ], + ) + for f in self._page_frames(first, end): + chunk = obj.data[f.offset : f.offset + f.length] + if mode == "host-read": + os.pwrite(fds[0], chunk, off + f.offset - first) + else: + self.gpu_write(fds[0], off + f.offset - first, chunk) + conn.sendall(wire.encode_frame(f.kind, f.number, f.offset, f.length)) + if self.truncate: + return + conn.sendall(wire.encode_frame(wire.COMPLETE_FRAME, pages, end - first, 0)) + return + body = end - first + wire.FRAME_SIZE * (pages + 1) + self._send(conn, 200, sel + [("Content-Length", str(body))]) + credits = rest or b"" + inflight = 0 + for i, f in enumerate(self._page_frames(first, end)): + number = f.number + (1 if self.bad_frame and i == 1 else 0) + conn.sendall(wire.encode_frame(f.kind, number, f.offset, f.length)) + conn.sendall(obj.data[f.offset : f.offset + f.length]) + inflight += 1 + if inflight >= 2: + credits = self._credit(conn, credits, seen) + inflight -= 1 + while inflight: + credits = self._credit(conn, credits, seen) + inflight -= 1 + if self.truncate: + return + conn.sendall(wire.encode_frame(wire.COMPLETE_FRAME, pages, end - first, 0)) + + def _credit(self, conn, buf, seen): + while len(buf) < wire.CREDIT_SIZE: + got = conn.recv(64) + if not got: + raise ConnectionError("client closed") + buf += got + seen.credits.append(wire.decode_credit(buf[: wire.CREDIT_SIZE])) + return buf[wire.CREDIT_SIZE :] + + @staticmethod + def _page_frames(first, end): + if end <= first: + return + n = first // wire.PAGE_SIZE + o = first + while o < end: + e = min(end, (n + 1) * wire.PAGE_SIZE) + yield wire.Frame(wire.PAGE_FRAME, n, o, e - o) + n += 1 + o = e diff --git a/clients/python/racer/tests/test_client.py b/clients/python/racer/tests/test_client.py new file mode 100644 index 000000000..0098f827a --- /dev/null +++ b/clients/python/racer/tests/test_client.py @@ -0,0 +1,319 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +import os + +import pytest +from conftest import FakeGpu +from fakeserver import Obj + +import racer +from racer import _wire as wire +from racer.client import _mode_state + +DATA = bytes(range(256)) * 50 # 12800 bytes: 4 pages of 4096 (last partial) + + +def add(server, data=DATA, etag='"v1"', name="obj"): + req = racer.Request(racer.object_request("bucket", name).key) + server.objects[req.hex] = Obj(data, etag, "text/plain") + return req + + +def client(server, **kw): + kw.setdefault("staging_bytes", 3 * 4096) + return racer.Client(socket_path=server.path, **kw) + + +def test_socket_path(monkeypatch): + monkeypatch.delenv("RACER_SOCKET", raising=False) + assert racer.socket_path_for("c1") == "/run/racer/c1/client/socket" + assert racer.socket_path_for("c1", "/x") == "/x" + monkeypatch.setenv("RACER_SOCKET", "/env") + assert racer.socket_path_for("c1") == "/env" + monkeypatch.delenv("RACER_SOCKET") + with pytest.raises(racer.RacerError): + racer.socket_path_for(None) + with pytest.raises(racer.RacerError): + racer.socket_path_for("a/b") + + +def test_stat(server): + req = add(server) + m = client(server).stat(req) + assert (m.etag, m.size, m.content_type, m.expires_at_ms) == ('"v1"', len(DATA), "text/plain", 1_700_000_000_000) + seen = server.seen[-1] + assert seen.method == "HEAD" and seen.headers["host"] == "racer" + assert seen.headers["racer-metadata"] == req.metadata if req.metadata else True + with pytest.raises(racer.VersionUnavailable): + client(server).stat(req, if_match='"v2"') + with pytest.raises(racer.NotFound): + client(server).stat(racer.Request(b"\0" * 32)) + + +def test_stream_read_full_and_range(server): + req = add(server) + c = client(server) + assert c.read(req) == DATA + seen = server.seen[-1] + assert seen.headers["racer-page-credits"] == "2" + assert seen.headers["racer-byte-credits"] == str(2 * 16 * 1024 * 1024) + assert "range" not in seen.headers + # Every page's credit comes back, including the last one. + assert seen.credits == [(0, 4096), (1, 4096), (2, 4096), (3, 512)] + assert c.read(req, offset=100, length=5000) == DATA[100:5100] + assert server.seen[-1].headers["range"] == "bytes=100-5099" + assert c.read(req, offset=5000) == DATA[5000:] + assert server.seen[-1].headers["range"] == "bytes=5000-" + + +def test_stream_empty_object(server): + req = add(server, data=b"") + assert client(server).read(req) == b"" + + +def test_stream_short_object_is_unsatisfiable(server): + req = add(server) + with pytest.raises(racer.UnsatisfiableRange): + client(server).read(req, offset=len(DATA) - 10, length=100) + with pytest.raises(racer.UnsatisfiableRange): + client(server).read(req, offset=len(DATA) + 1) + + +def test_stream_bad_frame(server): + req = add(server) + server.bad_frame = True + with pytest.raises(racer.ProtocolError): + client(server).read(req) + + +def test_stream_truncated(server): + req = add(server) + server.truncate = True + with pytest.raises(racer.TruncatedError): + client(server).read(req) + + +def test_if_match_sent(server): + req = add(server) + assert client(server).read(req, if_match='"v1"') == DATA + assert server.seen[-1].headers["if-match"] == '"v1"' + with pytest.raises(racer.VersionUnavailable): + client(server).read(req, if_match='"v0"') + with pytest.raises(racer.RacerError): + client(server).read(req, if_match="unquoted") + + +def test_host_read_passes_memfd(server): + req = add(server) + pages = [] + with racer.HostBuffer(len(DATA) + 10) as buf: + r = client(server).read_into_host(req, buf, 10, on_page=lambda o, n: pages.append((o, n))) + assert r.delivery == racer.HOST and r.pages == 4 + assert bytes(buf.view[10:]) == DATA + seen = server.seen[-1] + assert seen.target.endswith("/host-read") + assert seen.fds == 1 and seen.fd_kind == "regular" + assert seen.headers["racer-destination-offset"] == "10" + assert seen.headers["racer-destination-length"] == str(len(DATA)) + assert "racer-allocation-size" not in seen.headers + assert pages == [(0, 4096), (4096, 4096), (8192, 4096), (12288, 512)] + + +def test_host_read_range(server): + req = add(server) + with racer.HostBuffer(5000) as buf: + r = client(server).read_into_host(req, buf, offset=4000, length=5000) + assert (r.first, r.end) == (4000, 9000) + assert bytes(buf.view) == DATA[4000:9000] + + +def test_old_dataplane_falls_back_to_stream_and_remembers(server): + req = add(server) + server.old = True + c = client(server) + with racer.HostBuffer(len(DATA)) as buf: + r = c.read_into_host(req, buf) + assert r.delivery == racer.STREAM + assert bytes(buf.view) == DATA + assert c.mode_state(racer.HOST) == "unsupported" + n = len(server.seen) + r = c.read_into_host(req, buf) + assert r.delivery == racer.STREAM + # The second read goes straight to stream mode. + assert len(server.seen) == n + 1 and "/host-read" not in server.seen[-1].target + + +def test_missing_object_does_not_disable_mode(server): + c = client(server) + with racer.HostBuffer(10) as buf: + with pytest.raises(racer.NotFound): + c.read_into_host(racer.Request(b"\1" * 32), buf) + assert c.mode_state(racer.HOST) is None + + +def test_supported_mode_errors_are_not_fallbacks(server): + req = add(server) + c = client(server) + with racer.HostBuffer(len(DATA)) as buf: + c.read_into_host(req, buf) + assert c.mode_state(racer.HOST) == "supported" + server.host_status = 400 + with pytest.raises(racer.StatusError): + c.read_into_host(req, buf) + + +def test_plain_buffer_uses_stream(server): + req = add(server) + out = bytearray(len(DATA)) + r = client(server).read_into_host(req, out) + assert r.delivery == racer.STREAM and bytes(out) == DATA + assert server.seen[-1].fds == 0 + + +def test_destination_too_small(server): + req = add(server) + with racer.HostBuffer(100) as buf: + # host-read answers 400, which looks like an old dataplane, so the + # stream fallback reports the size mismatch. + with pytest.raises(racer.RacerError, match="smaller"): + client(server).read_into_host(req, buf) + out = bytearray(100) + with pytest.raises(racer.RacerError): + client(server).read_into_host(req, out) + + +def test_gpu_read_success(server, gpu_writer): + req = add(server) + server.gpu_status = 200 + server.gpu_write = gpu_writer + g = FakeGpu(len(DATA) + 100) + try: + r = client(server).read_into_gpu(req, g, 100) + assert r.delivery == racer.GPU + assert g.read(100, len(DATA)) == DATA + seen = server.seen[-1] + assert seen.target.endswith("/gpu-read") and seen.fds == 1 + assert seen.headers["racer-allocation-size"] == str(g.size) + assert seen.headers["racer-destination-length"] == str(len(DATA)) + assert g.copies == 0 + finally: + g.close() + + +def test_gpu_falls_back_to_host_chunks(server): + req = add(server) + g = FakeGpu(len(DATA)) + pages = [] + try: + c = client(server) + r = c.read_into_gpu(req, g, on_page=lambda o, n: pages.append((o, n))) + assert r.delivery == racer.HOST and (r.first, r.end, r.pages) == (0, len(DATA), 4) + assert g.read(0, len(DATA)) == DATA + assert c.mode_state(racer.GPU) == "unsupported" + assert pages == [(0, 4096), (4096, 4096), (8192, 4096), (12288, 512)] + reads = [s for s in server.seen if s.target.endswith("/host-read")] + # Staging holds 3 pages, so the object takes two host-reads. + assert len(reads) == 2 and reads[1].headers["if-match"] == '"v1"' + finally: + g.close() + + +def test_gpu_falls_back_to_stream_on_old_dataplane(server): + req = add(server) + server.old = True + g = FakeGpu(len(DATA)) + try: + c = client(server) + r = c.read_into_gpu(req, g, offset=10, length=9000) + assert r.delivery == racer.STREAM + assert g.read(0, 9000) == DATA[10:9010] + assert c.mode_state(racer.GPU) == "unsupported" + assert c.mode_state(racer.HOST) == "unsupported" + finally: + g.close() + + +def test_gpu_not_exportable_skips_without_memory(server): + req = add(server) + g = FakeGpu(len(DATA), exportable=False) + try: + c = client(server) + r = c.read_into_gpu(req, g) + assert r.delivery == racer.HOST + assert c.mode_state(racer.GPU) is None + finally: + g.close() + + +def test_modes_remembered_per_socket(server, tmp_path): + req = add(server) + server.old = True + with racer.HostBuffer(len(DATA)) as buf: + client(server).read_into_host(req, buf) + assert _mode_state[server.path][racer.HOST] == "unsupported" + other = racer.Client(socket_path=str(tmp_path / "other")) + assert other.mode_state(racer.HOST) is None + + +def test_send_fd_with_head(server): + """The descriptor arrives with the head even when nothing else is sent.""" + req = add(server) + with racer.HostBuffer(len(DATA)) as buf: + c = client(server) + c.host_read(req, buf.fileno(), 0, len(DATA)) + assert server.seen[-1].fds == 1 + assert bytes(buf.view) == DATA + # Our copy of the descriptor is still open and owned by the caller. + fd = os.memfd_create("x") + c.host_read(req, fd, 0, len(DATA)) + os.fstat(fd) + os.close(fd) + + +def test_gpu_read_window_check(server): + req = add(server) + with pytest.raises(racer.RacerError): + client(server).gpu_read(req, 0, 100, 50, 60) + + +def test_frame_sequence_rules(small_pages): + s = wire.Sequence(100, 9000) + with pytest.raises(racer.ProtocolError): + s.accept(wire.Frame(1, 0, 0, 4096)) # starts before range + s = wire.Sequence(100, 9000) + s.accept(wire.Frame(1, 0, 100, 3996)) + with pytest.raises(racer.ProtocolError): + s.accept(wire.Frame(1, 2, 8192, 808)) # skipped a page + s = wire.Sequence(100, 9000) + s.accept(wire.Frame(1, 0, 100, 3996)) + s.accept(wire.Frame(1, 1, 4096, 4096)) + with pytest.raises(racer.ProtocolError): + s.accept(wire.Frame(2, 3, 8900, 0)) # completion too early + s.accept(wire.Frame(1, 2, 8192, 808)) + with pytest.raises(racer.ProtocolError): + s.accept(wire.Frame(2, 3, 8900, 1)) # nonzero length + s.accept(wire.Frame(2, 3, 8900, 0)) + with pytest.raises(racer.ProtocolError): + s.accept(wire.Frame(2, 3, 8900, 0)) # after completion + assert wire.decode_frame(wire.encode_frame(1, 2, 3, 4)) == wire.Frame(1, 2, 3, 4) + assert wire.encode_credit(1, 2) == b"\0" * 7 + b"\1" + b"\0\0\0\2" + + +def test_decimal(): + assert wire.decimal("0") == 0 + for bad in ["", "01", "-1", "1a", "9" * 20, str(1 << 63)]: + with pytest.raises(racer.ProtocolError): + wire.decimal(bad) + + +def test_response_head_parsing(): + h = wire.parse_response_head(b"HTTP/1.1 200 OK\r\nETag: \"a\"\r\nConnection: keep-alive, close\r\n\r\n") + assert h.status == 200 and h.get("ETag") == '"a"' and wire.connection_close(h) + with pytest.raises(racer.ProtocolError): + wire.parse_response_head(b"HTTP/1.1 200 OK\r\nETag: a\r\nETag: b\r\n\r\n") + with pytest.raises(racer.ProtocolError): + wire.parse_response_head(b"HTTP/1.0 200 OK\r\n\r\n") + with pytest.raises(racer.ProtocolError): + wire.check_error_status(wire.parse_response_head(b"HTTP/1.1 404 N\r\nContent-Length: 3\r\n\r\n")) + with pytest.raises(racer.ProtocolError): + wire.check_error_status(wire.parse_response_head(b"HTTP/1.1 416 N\r\nContent-Length: 0\r\n\r\n")) diff --git a/clients/python/racer/tests/test_cuda.py b/clients/python/racer/tests/test_cuda.py new file mode 100644 index 000000000..6319f6b3e --- /dev/null +++ b/clients/python/racer/tests/test_cuda.py @@ -0,0 +1,51 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +import ctypes + +import pytest + +from racer import cuda + + +def test_struct_layouts(): + # Sizes and offsets of the cuda.h structs on LP64 Linux. + assert ctypes.sizeof(cuda.CUmemLocation) == 8 + assert ctypes.sizeof(cuda.CUmemAllocationProp) == 32 + assert cuda.CUmemAllocationProp.location.offset == 8 + assert cuda.CUmemAllocationProp.win32HandleMetaData.offset == 16 + assert cuda.CUmemAllocationProp.allocFlags.offset == 24 + assert ctypes.sizeof(cuda.CUmemAccessDesc) == 12 + + +def test_missing_libcuda(monkeypatch): + monkeypatch.setenv("RACER_LIBCUDA", "libdoes-not-exist.so") + monkeypatch.setattr(cuda, "_lib", None) + assert not cuda.available() + with pytest.raises(cuda.CudaUnavailable): + cuda.GpuAllocation(1) + + +def test_host_buffer_pin_without_cuda(monkeypatch): + from racer import HostBuffer + + monkeypatch.setenv("RACER_LIBCUDA", "libdoes-not-exist.so") + monkeypatch.setattr(cuda, "_lib", None) + with HostBuffer(4096) as b: + assert b.pin() is False and not b.pinned + b.view[:3] = b"abc" + assert bytes(b.view[:3]) == b"abc" + + +@pytest.mark.skipif(not cuda.available(), reason="needs a GPU") +def test_gpu_allocation_roundtrip(): + with cuda.GpuAllocation(1000) as g: + assert g.size >= 1000 + g.copy_from_host(0, b"hello") + out = bytearray(5) + g.copy_to_host(0, out) + assert out == b"hello" + import os + + fd = g.export_fd() + assert fd >= 0 + os.close(fd) diff --git a/clients/python/racer/tests/test_keys.py b/clients/python/racer/tests/test_keys.py new file mode 100644 index 000000000..912bf8957 --- /dev/null +++ b/clients/python/racer/tests/test_keys.py @@ -0,0 +1,63 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +import pytest + +import racer + +# Computed with internal/racerobject.NewRequest (Go encoding/json + SHA-256). +VECTORS = [ + ( + "models", + "llama/config.json", + "", + "3af125b4fe06cdb5fc810214a357c138d4acfc54d99ef56791f805182d4044e0", + '{"schema":1,"bucket":"models","key":"llama/config.json"}', + ), + ( + "models", + "llama/model-00001-of-00002.safetensors", + "v1", + "982a88e7fe93c3d53ff66b9045aa99089251f66d613790c6c5888b97054bdd16", + '{"schema":1,"bucket":"models","key":"llama/model-00001-of-00002.safetensors","versionId":"v1"}', + ), + ( + "b.u-c_k", + 'a&c "q" \\ \n\t\b\f\x01\x7f \u00e9 \u65e5\u672c \u2028\u2029 \U0001f600', + "3HL4kqtJlcpXroDTDmJ+rmSpXd3dIbrHY+MTRCxf3vjVBH40Nr8X8gdRQBpUMLUo", + "fde698046567709f2f289be4e4dc246b9fbc40a04b4df1ca0f3cf2d6e891b1ac", + None, + ), +] + + +@pytest.mark.parametrize("bucket,key,version,want,canonical", VECTORS) +def test_object_key_vectors(bucket, key, version, want, canonical): + assert racer.object_key(bucket, key, version) == want + if canonical is not None: + assert racer.canonical_json(bucket, key, version) == canonical + r = racer.object_request(bucket, key, version or None) + assert r.hex == want and r.metadata == racer.canonical_json(bucket, key, version) + + +@pytest.mark.parametrize( + "bucket,key,version", + [ + ("", "k", ""), + (".", "k", ""), + ("a/b", "k", ""), + ("b", "", ""), + ("b", "x" * 1025, ""), + ("b", "k", "v" * 1025), + ("b", "\ud800", ""), + ], +) +def test_object_key_invalid(bucket, key, version): + with pytest.raises(racer.RacerError): + racer.object_key(bucket, key, version) + + +def test_request_from_hex(): + r = racer.Request.from_hex("ab" * 32) + assert r.key == b"\xab" * 32 + with pytest.raises(racer.RacerError): + racer.Request.from_hex("AB" * 32) From 3ae47c031a6013fb596d5f9638fe622b25b19a4a Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 18:34:36 +0000 Subject: [PATCH 03/87] feat(racer): add racer-vllm loader plugin, test target and CI job --- .github/workflows/ci.yaml | 11 + Makefile | 9 + clients/python/racer-vllm/README.md | 75 ++++++ clients/python/racer-vllm/pyproject.toml | 27 +++ .../racer-vllm/src/racer_vllm/__init__.py | 8 + .../racer-vllm/src/racer_vllm/loader.py | 95 ++++++++ .../python/racer-vllm/src/racer_vllm/model.py | 168 +++++++++++++ .../racer-vllm/src/racer_vllm/safetensors.py | 157 +++++++++++++ .../racer-vllm/src/racer_vllm/weights.py | 220 ++++++++++++++++++ clients/python/racer-vllm/tests/conftest.py | 33 +++ .../python/racer-vllm/tests/test_loader.py | 188 +++++++++++++++ .../racer-vllm/tests/test_safetensors.py | 87 +++++++ 12 files changed, 1078 insertions(+) create mode 100644 clients/python/racer-vllm/README.md create mode 100644 clients/python/racer-vllm/pyproject.toml create mode 100644 clients/python/racer-vllm/src/racer_vllm/__init__.py create mode 100644 clients/python/racer-vllm/src/racer_vllm/loader.py create mode 100644 clients/python/racer-vllm/src/racer_vllm/model.py create mode 100644 clients/python/racer-vllm/src/racer_vllm/safetensors.py create mode 100644 clients/python/racer-vllm/src/racer_vllm/weights.py create mode 100644 clients/python/racer-vllm/tests/conftest.py create mode 100644 clients/python/racer-vllm/tests/test_loader.py create mode 100644 clients/python/racer-vllm/tests/test_safetensors.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6e0d5e224..b919ba578 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -392,6 +392,17 @@ jobs: - name: Provision envtest and run controller API-server race tests run: timeout --signal=TERM --kill-after=10s 300s make racer-envtest-ci + racer-python: + name: Racer Python Client + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Run Python client and vLLM plugin tests + run: timeout --signal=TERM --kill-after=10s 540s make racer-python-test + # ---------- Orca Integration Tests ---------- # Spins up Garage and Azurite via testcontainers-go and runs the # orca in-process integration suite (internal/orca/inttest). Docker diff --git a/Makefile b/Makefile index 91b0b53a7..8cfade985 100644 --- a/Makefile +++ b/Makefile @@ -581,6 +581,15 @@ racer-server-test: ## Lint and race-test the Racer server racer-test: racer-server-test ## Check the Racer controller +RACER_PY_VENV ?= $(CURDIR)/tmp/racer-python-venv + +.PHONY: racer-python-test +racer-python-test: ## Run the Racer Python client and vLLM plugin tests (no GPU needed) + @test -x "$(RACER_PY_VENV)/bin/pytest" || { timeout --signal=TERM --kill-after=10s 120s python3 -m venv "$(RACER_PY_VENV)" && \ + timeout --signal=TERM --kill-after=10s 300s "$(RACER_PY_VENV)/bin/pip" install -q pytest; } + cd clients/python/racer && PYTHONPATH=src timeout --signal=TERM --kill-after=10s 300s "$(RACER_PY_VENV)/bin/pytest" -q -p no:cacheprovider + cd clients/python/racer-vllm && PYTHONPATH=src:../racer/src timeout --signal=TERM --kill-after=10s 300s "$(RACER_PY_VENV)/bin/pytest" -q -p no:cacheprovider + $(SETUP_ENVTEST): @mkdir -p bin tmp/envtest-tools TMPDIR="$(CURDIR)/tmp/envtest-tools" GOBIN="$(CURDIR)/bin" timeout --signal=TERM --kill-after=10s 300s $(GOCMD) install sigs.k8s.io/controller-runtime/tools/setup-envtest@$(SETUP_ENVTEST_VERSION) diff --git a/clients/python/racer-vllm/README.md b/clients/python/racer-vllm/README.md new file mode 100644 index 000000000..d28a76cc8 --- /dev/null +++ b/clients/python/racer-vllm/README.md @@ -0,0 +1,75 @@ +# racer-vllm + +A vLLM plugin that loads model weights from a Racer cache. It adds the load +format `racer`. It uses the `racer-client` package. + +## Install + +``` +pip install ./clients/python/racer ./clients/python/racer-vllm +``` + +vLLM finds the plugin through the `vllm.general_plugins` entry point. It was +written for vLLM v0.25.x. + +## Model location + +A model lives at `racer:////`. The plugin reads +objects through `/run/racer//client/socket`. Set `RACER_SOCKET` to use +another socket. Object keys follow racer-object, so `//` +names the S3 object. + +## Use + +vLLM reads the model config and tokenizer before it loads weights, so these +small files must be on local disk first. Fetch them, for example in an init +container: + +``` +racer-vllm-fetch racer://models/my-bucket/llama-3-8b /models/llama-3-8b +``` + +This copies `config.json`, tokenizer files, `model.safetensors.index.json` +and similar files, plus small files named in the index metadata. It also +writes `racer-model.json`, which tells the loader where the weights are. + +Then start vLLM on that directory: + +``` +vllm serve /models/llama-3-8b --load-format racer \ + --model-loader-extra-config '{"staging_bytes": 268435456, "concurrency": 2}' +``` + +Extra config keys, all optional: + +| Key | Default | Meaning | +|---|---|---| +| `uri` | from `racer-model.json` | The `racer://` model location. | +| `staging_bytes` | 256 MiB | Size of each GPU staging buffer. | +| `concurrency` | 2 | Staging buffers, and reads in flight. | +| `clone` | true | Copy each tensor out of staging before handing it to vLLM. | +| `socket` | none | Client socket path. | +| `use_vmm` | true | Allow exportable `cuMemCreate` staging, needed for `gpu-read`. | + +## How it loads + +1. Find the safetensors files from `model.safetensors.index.json`, or use + `model.safetensors`. +2. For each file, read the 8-byte header length and the JSON header. +3. Group tensors into ranges that fit a staging buffer. Read each range into + GPU memory with `gpu-read`. If the dataplane cannot do that, use + `host-read` into pinned memory and copy, else the normal stream. +4. Give each tensor (right dtype and shape) to `model.load_weights`. While + vLLM uses one buffer, the next range is read into another. + +A tensor bigger than a staging buffer gets its own GPU allocation. + +With `clone` set to false, tensors are views into staging memory. They stay +valid only until vLLM asks for the next tensors after the buffer's last one. +Only use this when the model copies each weight at once. + +## Tests + +``` +make racer-python-test +``` diff --git a/clients/python/racer-vllm/pyproject.toml b/clients/python/racer-vllm/pyproject.toml new file mode 100644 index 000000000..55540c18d --- /dev/null +++ b/clients/python/racer-vllm/pyproject.toml @@ -0,0 +1,27 @@ +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[project] +name = "racer-vllm" +version = "0.1.0" +description = "vLLM model loader that reads weights from a Racer cache" +readme = "README.md" +license = { text = "Apache-2.0" } +requires-python = ">=3.10" +dependencies = ["racer-client"] + +[project.optional-dependencies] +test = ["pytest"] + +[project.entry-points."vllm.general_plugins"] +racer = "racer_vllm:register" + +[project.scripts] +racer-vllm-fetch = "racer_vllm.model:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/clients/python/racer-vllm/src/racer_vllm/__init__.py b/clients/python/racer-vllm/src/racer_vllm/__init__.py new file mode 100644 index 000000000..f1801da28 --- /dev/null +++ b/clients/python/racer-vllm/src/racer_vllm/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""vLLM plugin that loads model weights from a Racer cache.""" + +from .loader import LOAD_FORMAT, register +from .model import ModelURI, fetch, parse_uri + +__all__ = ["LOAD_FORMAT", "ModelURI", "fetch", "parse_uri", "register"] diff --git a/clients/python/racer-vllm/src/racer_vllm/loader.py b/clients/python/racer-vllm/src/racer_vllm/loader.py new file mode 100644 index 000000000..2a4b08794 --- /dev/null +++ b/clients/python/racer-vllm/src/racer_vllm/loader.py @@ -0,0 +1,95 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""vLLM model loader for --load-format racer. + +Targets vLLM v0.25.x. The loader interface used is +vllm/model_executor/model_loader/base_loader.py (BaseModelLoader with +download_model(model_config) and load_weights(model, model_config)), and +registration uses register_model_loader from +vllm/model_executor/model_loader/__init__.py. Recent main adds optional hooks +(get_external_weight_memory, create_model) with defaults, so this loader +works there too. +""" + +from __future__ import annotations + +import logging + +from .model import client_for, resolve_uri, weight_files +from .weights import DEFAULT_DEPTH, DEFAULT_STAGING_BYTES, TorchBackend, iter_weights + +log = logging.getLogger(__name__) + +LOAD_FORMAT = "racer" +_registered = False + + +def _settings(extra: dict) -> dict: + return { + "staging_bytes": int(extra.get("staging_bytes", DEFAULT_STAGING_BYTES)), + "concurrency": int(extra.get("concurrency", DEFAULT_DEPTH)), + "clone": bool(extra.get("clone", True)), + "socket": extra.get("socket"), + "use_vmm": bool(extra.get("use_vmm", True)), + } + + +def make_loader_class(base): + class RacerModelLoader(base): + """Loads safetensors weights from a Racer cache. + + model_loader_extra_config keys: + uri racer:////; optional when the + model directory has racer-model.json (racer-vllm-fetch) + staging_bytes size of each GPU staging buffer (default 256 MiB) + concurrency number of staging buffers and reads in flight (default 2) + clone copy each tensor out of staging (default true) + socket client socket path (else RACER_SOCKET, else the cache) + use_vmm allow exportable cuMemCreate staging (default true) + """ + + def __init__(self, load_config): + super().__init__(load_config) + self.extra = dict(getattr(load_config, "model_loader_extra_config", None) or {}) + self.settings = _settings(self.extra) + + def _client(self, model_config): + uri = resolve_uri(str(model_config.model), self.extra) + client = client_for(uri, self.settings["socket"], staging_bytes=self.settings["staging_bytes"]) + return uri, client + + def download_model(self, model_config) -> None: + # Nothing to download; check the weights are reachable. + uri, client = self._client(model_config) + try: + weight_files(client, uri) + finally: + client.close() + + def load_weights(self, model, model_config) -> None: + uri, client = self._client(model_config) + s = self.settings + try: + backend = TorchBackend(clone=s["clone"], use_vmm=s["use_vmm"]) + weights = iter_weights(client, uri, backend, s["staging_bytes"], s["concurrency"]) + loaded = model.load_weights(weights) + finally: + client.close() + if loaded is not None and getattr(model_config, "quantization", None) is None: + missing = {n for n, _ in model.named_parameters()} - set(loaded) + if missing: + raise ValueError(f"racer loader: weights not loaded: {sorted(missing)[:10]}") + + return RacerModelLoader + + +def register() -> None: + """vllm.general_plugins entry point. Safe to call more than once.""" + global _registered + if _registered: + return + from vllm.model_executor.model_loader import register_model_loader + from vllm.model_executor.model_loader.base_loader import BaseModelLoader + + register_model_loader(LOAD_FORMAT)(make_loader_class(BaseModelLoader)) + _registered = True diff --git a/clients/python/racer-vllm/src/racer_vllm/model.py b/clients/python/racer-vllm/src/racer_vllm/model.py new file mode 100644 index 000000000..8eafc2dfb --- /dev/null +++ b/clients/python/racer-vllm/src/racer_vllm/model.py @@ -0,0 +1,168 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""racer:// model locations and small-file fetching.""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass + +import racer + +SCHEME = "racer://" +MARKER = "racer-model.json" +INDEX = "model.safetensors.index.json" +SINGLE = "model.safetensors" + +# Small files vLLM and transformers read to build the model and tokenizer. +CONFIG_FILES = ( + "config.json", + "generation_config.json", + "tokenizer.json", + "tokenizer_config.json", + "special_tokens_map.json", + "tokenizer.model", + "vocab.json", + "vocab.txt", + "merges.txt", + "added_tokens.json", + "chat_template.jinja", + "chat_template.json", + "preprocessor_config.json", + "processor_config.json", + "params.json", + "tekken.json", + INDEX, +) +REQUIRED_FILES = ("config.json",) +_SMALL_SUFFIXES = (".json", ".txt", ".model", ".jinja", ".py", ".tiktoken") + + +@dataclass(frozen=True) +class ModelURI: + cache: str + bucket: str + prefix: str # no leading or trailing slash; may be empty + + def key(self, name: str) -> str: + return f"{self.prefix}/{name}" if self.prefix else name + + def request(self, name: str) -> racer.Request: + return racer.object_request(self.bucket, self.key(name)) + + def __str__(self) -> str: + return f"{SCHEME}{self.cache}/{self.bucket}/{self.prefix}".rstrip("/") + + +def parse_uri(uri: str) -> ModelURI: + """Parse racer:////.""" + if not uri.startswith(SCHEME): + raise ValueError(f"not a racer:// URI: {uri!r}") + parts = uri[len(SCHEME) :].split("/", 2) + if len(parts) < 2 or not parts[0] or not parts[1]: + raise ValueError(f"racer URI needs a cache and a bucket: {uri!r}") + prefix = parts[2].strip("/") if len(parts) == 3 else "" + return ModelURI(parts[0], parts[1], prefix) + + +def client_for(uri: ModelURI, socket_path: str | None = None, **kw) -> racer.Client: + """Client for the URI's cache. RACER_SOCKET or socket_path override the path.""" + return racer.Client(cache=uri.cache, socket_path=socket_path, **kw) + + +def read_small(client: racer.Client, uri: ModelURI, name: str) -> bytes | None: + """Read a whole small object, or None when it does not exist.""" + try: + return client.read(uri.request(name)) + except racer.NotFound: + return None + + +def index_extra_files(index: dict) -> list: + """Small file names listed as string values in the index metadata.""" + out = [] + for v in (index.get("metadata") or {}).values(): + if isinstance(v, str) and v.endswith(_SMALL_SUFFIXES) and not v.startswith("/") and ".." not in v: + out.append(v) + return out + + +def weight_files(client: racer.Client, uri: ModelURI) -> list: + """List safetensors files from the index, else the single model.safetensors.""" + raw = read_small(client, uri, INDEX) + if raw is None: + client.stat(uri.request(SINGLE)) # raises NotFound with a clear key + return [SINGLE] + index = json.loads(raw) + files = sorted(set((index.get("weight_map") or {}).values())) + if not files: + raise ValueError(f"{INDEX} has an empty weight_map") + for f in files: + if f.startswith("/") or ".." in f.split("/"): + raise ValueError(f"unsafe file name in {INDEX}: {f!r}") + return files + + +def fetch(uri: str | ModelURI, dest: str, client: racer.Client | None = None, extra=()) -> list: + """Copy the small model files to dest and write the racer-model.json marker. + + Point vLLM at dest (--model dest, and --tokenizer dest if needed) with + --load-format racer. The loader reads the marker to find the weights. + """ + u = parse_uri(uri) if isinstance(uri, str) else uri + client = client or client_for(u) + os.makedirs(dest, exist_ok=True) + names = list(CONFIG_FILES) + list(extra) + fetched = [] + seen = set() + while names: + name = names.pop(0) + if name in seen: + continue + seen.add(name) + data = read_small(client, u, name) + if data is None: + if name in REQUIRED_FILES: + raise FileNotFoundError(f"{u.key(name)} not found in {u}") + continue + path = os.path.join(dest, name) + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = path + ".tmp" + with open(tmp, "wb") as f: + f.write(data) + os.replace(tmp, path) + fetched.append(name) + if name == INDEX: + names.extend(index_extra_files(json.loads(data))) + with open(os.path.join(dest, MARKER), "w") as f: + json.dump({"uri": str(u)}, f) + return fetched + + +def resolve_uri(model_path: str, extra: dict) -> ModelURI: + """Find the racer:// URI from extra config, the model name, or the marker.""" + if extra.get("uri"): + return parse_uri(extra["uri"]) + if model_path.startswith(SCHEME): + return parse_uri(model_path) + marker = os.path.join(model_path, MARKER) + if os.path.exists(marker): + with open(marker) as f: + return parse_uri(json.load(f)["uri"]) + raise ValueError( + "racer loader: no model URI; set model_loader_extra_config.uri or run racer-vllm-fetch into the model directory" + ) + + +def main(argv=None) -> int: + import argparse + + p = argparse.ArgumentParser(prog="racer-vllm-fetch", description="Fetch model config files from Racer.") + p.add_argument("uri", help="racer:////") + p.add_argument("dest", help="local directory") + p.add_argument("--file", action="append", default=[], help="extra small file to fetch") + a = p.parse_args(argv) + for name in fetch(a.uri, a.dest, extra=a.file): + print(name) + return 0 diff --git a/clients/python/racer-vllm/src/racer_vllm/safetensors.py b/clients/python/racer-vllm/src/racer_vllm/safetensors.py new file mode 100644 index 000000000..88d20cffa --- /dev/null +++ b/clients/python/racer-vllm/src/racer_vllm/safetensors.py @@ -0,0 +1,157 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Torch-free safetensors header parsing and read planning. + +Format: an 8-byte little-endian header length N, N bytes of JSON, then the +tensor data. Each JSON entry has dtype, shape and data_offsets [begin, end) +relative to the end of the header. "__metadata__" holds string metadata. +""" + +from __future__ import annotations + +import json +import math +import struct +from dataclasses import dataclass, field + +MAX_HEADER_BYTES = 100 * 1024 * 1024 + +# Bytes per element for each safetensors dtype. +DTYPE_SIZES = { + "BOOL": 1, + "U8": 1, + "I8": 1, + "F8_E4M3": 1, + "F8_E5M2": 1, + "F8_E8M0": 1, + "I16": 2, + "U16": 2, + "F16": 2, + "BF16": 2, + "I32": 4, + "U32": 4, + "F32": 4, + "I64": 8, + "U64": 8, + "F64": 8, +} + + +class SafetensorsError(ValueError): + pass + + +@dataclass(frozen=True) +class TensorInfo: + name: str + dtype: str + shape: tuple + start: int # absolute file offset + end: int + + @property + def nbytes(self) -> int: + return self.end - self.start + + @property + def itemsize(self) -> int: + return DTYPE_SIZES[self.dtype] + + +@dataclass +class Header: + data_start: int + tensors: list + metadata: dict = field(default_factory=dict) + + +def header_length(prefix: bytes) -> int: + if len(prefix) != 8: + raise SafetensorsError("short safetensors prefix") + (n,) = struct.unpack(" MAX_HEADER_BYTES: + raise SafetensorsError(f"bad safetensors header length {n}") + return n + + +def parse_header(raw: bytes, file_size: int | None = None) -> Header: + """Parse header JSON (without the 8-byte prefix) into sorted TensorInfo.""" + try: + doc = json.loads(raw.decode("utf-8").rstrip(" ")) + except (UnicodeDecodeError, ValueError) as exc: + raise SafetensorsError("bad safetensors header JSON") from exc + if not isinstance(doc, dict): + raise SafetensorsError("safetensors header is not an object") + data_start = 8 + len(raw) + metadata = doc.pop("__metadata__", None) or {} + tensors = [] + for name, spec in doc.items(): + try: + dtype = spec["dtype"] + shape = tuple(int(d) for d in spec["shape"]) + begin, end = (int(x) for x in spec["data_offsets"]) + except (KeyError, TypeError, ValueError) as exc: + raise SafetensorsError(f"bad entry for {name!r}") from exc + if dtype not in DTYPE_SIZES: + raise SafetensorsError(f"unsupported dtype {dtype} for {name!r}") + if any(d < 0 for d in shape) or begin < 0 or end < begin: + raise SafetensorsError(f"bad shape or offsets for {name!r}") + if end - begin != math.prod(shape) * DTYPE_SIZES[dtype]: + raise SafetensorsError(f"size mismatch for {name!r}") + tensors.append(TensorInfo(name, dtype, shape, data_start + begin, data_start + end)) + tensors.sort(key=lambda t: (t.start, t.end)) + prev = data_start + for t in tensors: + if t.start < prev: + raise SafetensorsError(f"overlapping tensor {t.name!r}") + prev = t.end + if file_size is not None and prev > file_size: + raise SafetensorsError("tensor data past end of file") + return Header(data_start, tensors, metadata) + + +@dataclass +class Chunk: + """A contiguous file range read with one request.""" + + start: int + end: int + tensors: list + dedicated: bool = False # larger than the staging buffer + + @property + def length(self) -> int: + return self.end - self.start + + +def plan_chunks(tensors: list, staging_bytes: int) -> list: + """Group sorted tensors into ranges of at most staging_bytes. + + A tensor bigger than staging_bytes gets its own dedicated chunk. Zero-size + tensors join the current chunk and need no bytes. + """ + chunks: list = [] + cur: Chunk | None = None + for t in tensors: + if t.nbytes > staging_bytes: + if cur is not None: + chunks.append(cur) + cur = None + chunks.append(Chunk(t.start, t.end, [t], dedicated=True)) + continue + if cur is not None and t.end - cur.start <= staging_bytes: + cur.tensors.append(t) + cur.end = max(cur.end, t.end) + continue + if cur is not None: + chunks.append(cur) + cur = Chunk(t.start, t.end, [t]) + if cur is not None: + chunks.append(cur) + return chunks + + +def tensor_bytes(buf, chunk: Chunk, t: TensorInfo) -> memoryview: + """Return the bytes of t inside a buffer holding chunk (torch-free).""" + rel = t.start - chunk.start + return memoryview(buf)[rel : rel + t.nbytes] diff --git a/clients/python/racer-vllm/src/racer_vllm/weights.py b/clients/python/racer-vllm/src/racer_vllm/weights.py new file mode 100644 index 000000000..f4bc4e2fe --- /dev/null +++ b/clients/python/racer-vllm/src/racer_vllm/weights.py @@ -0,0 +1,220 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Stream safetensors weights from Racer into GPU staging buffers. + +The pipeline is torch-free; a Backend supplies memory and tensor views. +""" + +from __future__ import annotations + +import collections +import logging +from concurrent.futures import ThreadPoolExecutor +from typing import Iterator + +import racer + +from .model import ModelURI, weight_files +from .safetensors import Chunk, TensorInfo, header_length, parse_header, plan_chunks + +log = logging.getLogger(__name__) + +DEFAULT_STAGING_BYTES = 256 * 1024 * 1024 +DEFAULT_DEPTH = 2 + + +class Backend: + """Memory and tensor views for the pipeline.""" + + def new_staging(self, size: int): + raise NotImplementedError + + def new_dedicated(self, size: int): + raise NotImplementedError + + def tensor(self, target, rel: int, info: TensorInfo, staged: bool): + raise NotImplementedError + + def done(self, target) -> None: + """The consumer is finished with every tensor from target.""" + + def wait(self, target) -> None: + """Block until work queued by the consumer on target has finished.""" + + +def _file_chunks(client: racer.Client, uri: ModelURI, staging_bytes: int): + for name in weight_files(client, uri): + req = uri.request(name) + meta = client.stat(req) + prefix = client.read(req, 0, 8, if_match=meta.etag) + n = header_length(prefix) + header = parse_header(client.read(req, 8, n, if_match=meta.etag), meta.size) + log.debug("racer: %s has %d tensors", name, len(header.tensors)) + for chunk in plan_chunks(header.tensors, staging_bytes): + yield req, meta.etag, chunk + + +def iter_weights( + client: racer.Client, + uri: ModelURI, + backend: Backend, + staging_bytes: int = DEFAULT_STAGING_BYTES, + depth: int = DEFAULT_DEPTH, +) -> Iterator[tuple]: + """Yield (name, tensor) for every tensor in the model. + + Up to ``depth`` chunks are read ahead, each into its own staging buffer, + so reads overlap with the consumer. A staging buffer is reused only after + the consumer has moved past all of its tensors and backend.wait() says + queued GPU work that reads it has finished. + """ + if depth < 1: + raise ValueError("depth must be at least 1") + free = [backend.new_staging(staging_bytes) for _ in range(depth)] + chunks = _file_chunks(client, uri, staging_bytes) + inflight: collections.deque = collections.deque() + pool = ThreadPoolExecutor(max_workers=depth, thread_name_prefix="racer-read") + + def read(req, etag, chunk: Chunk, target): + backend.wait(target) + if chunk.length: + client.read_into_gpu(req, target, 0, offset=chunk.start, length=chunk.length, if_match=etag) + return target + + def fill() -> bool: + while len(inflight) < depth: + nxt = next(chunks, None) + if nxt is None: + return bool(inflight) + req, etag, chunk = nxt + if chunk.dedicated: + target = backend.new_dedicated(chunk.length) + else: + target = free.pop() + inflight.append((pool.submit(read, req, etag, chunk, target), chunk)) + return True + + try: + while fill(): + fut, chunk = inflight.popleft() + target = fut.result() + for t in chunk.tensors: + yield t.name, backend.tensor(target, t.start - chunk.start, t, not chunk.dedicated) + if not chunk.dedicated: + backend.done(target) + free.append(target) + finally: + for fut, _ in inflight: + fut.cancel() + pool.shutdown(wait=True) + + +class MemoryBackend(Backend): + """Host memory backend: plain buffers, bytes views. Used for tests and CPU.""" + + class Target: + def __init__(self, size: int) -> None: + self.size = size + self.buf = bytearray(size) + + def copy_from_host(self, offset: int, data) -> None: + mv = memoryview(data).cast("B") + self.buf[offset : offset + len(mv)] = mv + + def new_staging(self, size: int): + return self.Target(size) + + def new_dedicated(self, size: int): + return self.Target(size) + + def tensor(self, target, rel, info, staged): + return bytes(target.buf[rel : rel + info.nbytes]) + + +TORCH_DTYPES = { + "BOOL": "bool", + "U8": "uint8", + "I8": "int8", + "F8_E4M3": "float8_e4m3fn", + "F8_E5M2": "float8_e5m2", + "F8_E8M0": "float8_e8m0fnu", + "I16": "int16", + "U16": "uint16", + "F16": "float16", + "BF16": "bfloat16", + "I32": "int32", + "U32": "uint32", + "F32": "float32", + "I64": "int64", + "U64": "uint64", + "F64": "float64", +} + + +class TorchBackend(Backend): + """CUDA backend. + + Staging buffers are exportable cuMemCreate allocations when the CUDA + driver allows it (so gpu-read works), else plain torch tensors (host or + stream mode plus a copy). Tensors from staging buffers are cloned unless + clone=False; then they are views that are valid only until the consumer + asks for the next chunk's tensors. + """ + + def __init__(self, device: int | None = None, clone: bool = True, use_vmm: bool = True) -> None: + import torch + + self.torch = torch + self.device = torch.cuda.current_device() if device is None else device + self.clone = clone + self.use_vmm = use_vmm + self._flat: dict = {} + self._events: dict = {} + + def _new(self, size: int): + from racer import cuda, tensors + + if self.use_vmm: + try: + alloc = cuda.GpuAllocation(size, self.device) + self._flat[id(alloc)] = tensors.as_tensor(alloc) + return alloc + except racer.RacerError as exc: + log.info("racer: exportable GPU memory unavailable (%s); using torch memory", exc) + self.use_vmm = False + t = self.torch.empty(size, dtype=self.torch.uint8, device=f"cuda:{self.device}") + target = tensors.TensorTarget(t) + self._flat[id(target)] = t + return target + + def new_staging(self, size): + return self._new(size) + + def new_dedicated(self, size): + return self._new(size) + + def tensor(self, target, rel, info, staged): + torch = self.torch + dtype = getattr(torch, TORCH_DTYPES[info.dtype]) + if info.nbytes == 0: + return torch.empty(info.shape, dtype=dtype, device=f"cuda:{self.device}") + flat = self._flat[id(target)] + if not staged: + # Dedicated memory: the tensor keeps it alive; drop our reference. + self._flat.pop(id(target), None) + raw = flat[rel : rel + info.nbytes] + if rel % info.itemsize: + raw = raw.clone() # realign; the clone owns its memory + elif staged and self.clone: + raw = raw.clone() + return raw.view(dtype).view(info.shape) + + def done(self, target): + ev = self.torch.cuda.Event() + ev.record() + self._events[id(target)] = ev + + def wait(self, target): + ev = self._events.pop(id(target), None) + if ev is not None: + ev.synchronize() diff --git a/clients/python/racer-vllm/tests/conftest.py b/clients/python/racer-vllm/tests/conftest.py new file mode 100644 index 000000000..ea8395a24 --- /dev/null +++ b/clients/python/racer-vllm/tests/conftest.py @@ -0,0 +1,33 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +import json +import os +import struct +import sys + +HERE = os.path.dirname(__file__) +sys.path.insert(0, os.path.join(HERE, "..", "..", "racer", "tests")) + +import importlib.util # noqa: E402 + +# Reuse the racer client's fake server fixtures without a module name clash. +_spec = importlib.util.spec_from_file_location( + "racer_test_fixtures", os.path.join(HERE, "..", "..", "racer", "tests", "conftest.py") +) +_fixtures = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_fixtures) +server = _fixtures.server +small_pages = _fixtures.small_pages + + +def make_safetensors(tensors, metadata=None, pad=0): + """tensors: list of (name, dtype, shape, bytes). Returns file bytes.""" + header = {} + data = b"" + for name, dtype, shape, raw in tensors: + header[name] = {"dtype": dtype, "shape": list(shape), "data_offsets": [len(data), len(data) + len(raw)]} + data += raw + if metadata: + header["__metadata__"] = metadata + h = json.dumps(header).encode() + b" " * pad + return struct.pack(" Date: Thu, 22 Oct 2026 18:31:22 +0000 Subject: [PATCH 04/87] feat(racer): inject NVIDIA driver libraries into the dataplane --- deploy/racer/config.yaml.tmpl | 5 + deploy/racer/dataplane-config.yaml.tmpl | 5 + .../components/racer/dataplane_test.go | 6 + internal/racer/members/members_test.go | 4 +- internal/racer/workload/workload.go | 69 +++++++++- internal/racer/workload/workload_test.go | 121 ++++++++++++++++++ 6 files changed, 207 insertions(+), 3 deletions(-) create mode 100644 internal/racer/workload/workload_test.go diff --git a/deploy/racer/config.yaml.tmpl b/deploy/racer/config.yaml.tmpl index 9f6ceff21..9a4002354 100644 --- a/deploy/racer/config.yaml.tmpl +++ b/deploy/racer/config.yaml.tmpl @@ -31,6 +31,11 @@ data: # Discovery and authentication of the operator-managed dataplane workload. RACER_DATAPLANE_SERVICE_ACCOUNT: "racer-dataplane" RACER_DAEMONSET_NAME: "racer-dataplane" + # NVIDIA driver libraries for the dataplane: off, env (default), or cdi. + # env sets NVIDIA_VISIBLE_DEVICES=all and NVIDIA_DRIVER_CAPABILITIES=compute,utility. + # cdi also adds the Pod annotation cdi.k8s.io/racer-dataplane: nvidia.com/gpu=all; + # containerd then fails the Pod on any node without an NVIDIA CDI spec. + # RACER_DATAPLANE_NVIDIA: "env" RACER_CREDENTIALS_SECRET_NAME: {{ .CredentialsSecretName | default "racer-credentials" | quote }} RACER_VERSION_CONFIGMAP_NAME: {{ .VersionConfigMapName | default "racer-version" | quote }} RACER_INSTALLATION_CONFIGMAP_NAME: {{ .InstallationConfigMapName | default "racer-installation" | quote }} diff --git a/deploy/racer/dataplane-config.yaml.tmpl b/deploy/racer/dataplane-config.yaml.tmpl index 2215e0612..01c385369 100644 --- a/deploy/racer/dataplane-config.yaml.tmpl +++ b/deploy/racer/dataplane-config.yaml.tmpl @@ -13,6 +13,11 @@ data: # RDMA requires the same nonempty Site and rail on both peers. NUMA is # autodetected; same-rail candidates prefer local, then unknown, then remote. RACER_ENABLE_RDMA: "auto" + # GPU reads need an image built with RACER_NATIVE_CUDA=true. auto turns GPU + # mode off when no usable GPU is found; true fails startup instead. + # RACER_ENABLE_GPU: "auto" + # Optional cap on ciphertext staging memory per GPU, in bytes. + # RACER_GPU_STAGING_BYTES: "268435456" # No thread cap by default: eligible physical cores, CPU quota, and resource budgets apply. # I/O shards share NUMA-local crypto threads at approximately 2:1, using leftovers: # eight eligible cores on one NUMA node give five I/O plus three crypto threads diff --git a/internal/operator/components/racer/dataplane_test.go b/internal/operator/components/racer/dataplane_test.go index 6073589aa..b7636f357 100644 --- a/internal/operator/components/racer/dataplane_test.go +++ b/internal/operator/components/racer/dataplane_test.go @@ -94,6 +94,9 @@ func TestClusterCacheDeploysDataplane(t *testing.T) { require.Equal(t, marker.Data["cluster"], values["RACER_CLUSTER_ID"]) require.Equal(t, "https://racer-controller.custom-system.svc:8443", values["RACER_CONTROL_ENDPOINT"]) require.Equal(t, "[$(RACER_POD_IP)]:8082", values["RACER_PEER_LISTEN"]) + require.Equal(t, "all", values["NVIDIA_VISIBLE_DEVICES"]) + require.Equal(t, "compute,utility", values["NVIDIA_DRIVER_CAPABILITIES"]) + require.NotContains(t, ds.Spec.Template.Annotations, "cdi.k8s.io/racer-dataplane") var bootstrap *corev1.Volume @@ -147,9 +150,11 @@ func TestDataplaneWorkloadInputsFromRacerConfig(t *testing.T) { require.NoError(t, env.Client.Get(t.Context(), objectKey(env, configName), cm)) cm.Data["RACER_HOST_NETWORK"] = "true" cm.Data["RACER_DIAGNOSTICS_PORT"] = "9300" + cm.Data["RACER_DATAPLANE_NVIDIA"] = "cdi" require.NoError(t, env.Client.Update(t.Context(), cm)) persist(t, env, planPass(t, env)) + require.Equal(t, "nvidia.com/gpu=all", liveDataplane(t, env).Spec.Template.Annotations["cdi.k8s.io/racer-dataplane"]) pod := liveDataplane(t, env).Spec.Template.Spec require.True(t, pod.HostNetwork) require.Equal(t, corev1.DNSClusterFirstWithHostNet, pod.DNSPolicy) @@ -159,6 +164,7 @@ func TestDataplaneWorkloadInputsFromRacerConfig(t *testing.T) { {"RACER_POD_NETWORK_NODES", `["node-a"]`, "not supported"}, {"RACER_HOST_NETWORK", "yes", "RACER_HOST_NETWORK"}, {"RACER_DIAGNOSTICS_PORT", "80", "RACER_DIAGNOSTICS_PORT"}, + {"RACER_DATAPLANE_NVIDIA", "auto", "RACER_DATAPLANE_NVIDIA"}, } { t.Run(tc.key, func(t *testing.T) { require.NoError(t, env.Client.Get(t.Context(), objectKey(env, configName), cm)) diff --git a/internal/racer/members/members_test.go b/internal/racer/members/members_test.go index 9025ebebf..6475b8855 100644 --- a/internal/racer/members/members_test.go +++ b/internal/racer/members/members_test.go @@ -303,6 +303,8 @@ func TestWorkloadDataplaneEnvironment(t *testing.T) { "RACER_IDENTITY_DIRECTORY": "/var/lib/racer/identity/private", "RACER_SLAB_DIRECTORY": "/var/lib/racer/slabs", "RACER_DEVICE_DIRECTORY": "/host/dev", + "NVIDIA_VISIBLE_DEVICES": "all", + "NVIDIA_DRIVER_CAPABILITIES": "compute,utility", } if len(env) != len(expected) { t.Fatalf("unexpected configuration: %v", env) @@ -571,7 +573,7 @@ func TestWorkloadConfigIgnoresControllerRuntime(t *testing.T) { } switch key { - case "RACER_PEER_PORT", "RACER_HOST_NETWORK", "RACER_POD_NETWORK_NODES", "RACER_DIAGNOSTICS_PORT", "RACER_DATAPLANE_SERVICE_ACCOUNT", "RACER_DAEMONSET_NAME", "RACER_BOOTSTRAP_TRUST_CONFIGMAP": + case "RACER_PEER_PORT", "RACER_HOST_NETWORK", "RACER_POD_NETWORK_NODES", "RACER_DIAGNOSTICS_PORT", "RACER_DATAPLANE_SERVICE_ACCOUNT", "RACER_DAEMONSET_NAME", "RACER_BOOTSTRAP_TRUST_CONFIGMAP", "RACER_DATAPLANE_NVIDIA": return "", false default: t.Errorf("workload parser requested runtime setting %s", key) diff --git a/internal/racer/workload/workload.go b/internal/racer/workload/workload.go index 161ddc06f..b28ed66ec 100644 --- a/internal/racer/workload/workload.go +++ b/internal/racer/workload/workload.go @@ -31,6 +31,31 @@ const ( PodNetworkDaemonSetName = "racer-dataplane-podnet" ) +// NVIDIAInjection selects how the dataplane asks the container runtime for +// the host NVIDIA driver libraries. No mode requests nvidia.com/gpu. +type NVIDIAInjection string + +const ( + // NVIDIAOff adds nothing. + NVIDIAOff NVIDIAInjection = "off" + // NVIDIAEnv sets NVIDIA_VISIBLE_DEVICES and NVIDIA_DRIVER_CAPABILITIES. + // The NVIDIA container runtime acts on them; runc ignores them, so they + // are harmless on nodes without NVIDIA. + NVIDIAEnv NVIDIAInjection = "env" + // NVIDIACDI also adds the CDI pod annotation. containerd fails container + // creation when a CDI device cannot be resolved, so use this only when + // every dataplane node has an NVIDIA CDI spec. + NVIDIACDI NVIDIAInjection = "cdi" +) + +const ( + // CDIAnnotation requests the NVIDIA CDI device on the dataplane Pod. + // containerd treats every key with the cdi.k8s.io/ prefix as a CDI request. + CDIAnnotation = "cdi.k8s.io/racer-dataplane" + // CDIAllGPUs is the fully qualified CDI name for every GPU on the node. + CDIAllGPUs = "nvidia.com/gpu=all" +) + // ManagedNames contains only the explicitly configured workload. func ManagedNames(daemonSetName string) []string { return []string{daemonSetName} @@ -52,6 +77,8 @@ type Config struct { DiagnosticsPort uint16 DataplaneServiceAccount string DaemonSetName string + // Empty selects NVIDIAEnv. + NVIDIA NVIDIAInjection } // ConfigFromLookup reads operator deployment wiring without loading or @@ -75,6 +102,15 @@ func ConfigFromLookup(lookup func(string) (string, bool)) (Config, error) { return Config{}, fmt.Errorf("RACER_HOST_NETWORK must be true or false: %w", wire.InvalidRequest) } + // Unset keeps the zero value, which selects NVIDIAEnv. + var nvidia NVIDIAInjection + if value, ok := lookup("RACER_DATAPLANE_NVIDIA"); ok { + nvidia = NVIDIAInjection(value) + if nvidia != NVIDIAOff && nvidia != NVIDIAEnv && nvidia != NVIDIACDI { + return Config{}, fmt.Errorf("RACER_DATAPLANE_NVIDIA must be off, env, or cdi: %w", wire.InvalidRequest) + } + } + var diagnosticsPort uint64 if value, ok := lookup("RACER_DIAGNOSTICS_PORT"); ok { diagnosticsPort, err = strconv.ParseUint(value, 10, 16) @@ -102,6 +138,7 @@ func ConfigFromLookup(lookup func(string) (string, bool)) (Config, error) { DiagnosticsPort: uint16(diagnosticsPort), DataplaneServiceAccount: env("RACER_DATAPLANE_SERVICE_ACCOUNT", "racer-dataplane"), DaemonSetName: env("RACER_DAEMONSET_NAME", "racer-dataplane"), + NVIDIA: nvidia, } return cfg, cfg.Validate() @@ -116,9 +153,21 @@ func (c Config) Validate() error { return err } + if mode := c.nvidiaMode(); mode != NVIDIAOff && mode != NVIDIAEnv && mode != NVIDIACDI { + return fmt.Errorf("NVIDIA injection must be off, env, or cdi: %w", wire.InvalidRequest) + } + return c.validateEndpoint() } +func (c Config) nvidiaMode() NVIDIAInjection { + if c.NVIDIA == "" { + return NVIDIAEnv + } + + return c.NVIDIA +} + func (c Config) validateNetworkNodes() error { if len(c.PodNetworkNodes) != 0 && (!c.HostNetwork || c.DaemonSetName != DataplaneDaemonSetName) { return fmt.Errorf("pod network exceptions require host networking and the fixed dataplane name: %w", wire.InvalidRequest) @@ -186,6 +235,11 @@ func buildDaemonSet(c Config) *appsv1.DaemonSet { // Use workload identity, not the manager's identity, for the Pod selector. labels := map[string]string{"app.kubernetes.io/name": "racer-dataplane", "app.kubernetes.io/instance": c.DaemonSetName} + var annotations map[string]string + if c.nvidiaMode() == NVIDIACDI { + annotations = map[string]string{CDIAnnotation: CDIAllGPUs} + } + return &appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "DaemonSet"}, ObjectMeta: metav1.ObjectMeta{Name: c.DaemonSetName, Namespace: c.Namespace, Labels: labels}, @@ -200,7 +254,7 @@ func buildDaemonSet(c Config) *appsv1.DaemonSet { }, // Require sustained readiness across probe periods before advancing a rollout. MinReadySeconds: 10, - Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: dataplanePod(c)}, + Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: annotations}, Spec: dataplanePod(c)}, }, } } @@ -356,7 +410,7 @@ func dataplaneContainer(c Config) corev1.Container { } func dataplaneEnvironment(c Config, diagnosticsPort int32) []corev1.EnvVar { - return []corev1.EnvVar{ + env := []corev1.EnvVar{ { Name: "RACER_CLUSTER_ID", Value: string(c.Cluster), @@ -407,6 +461,17 @@ func dataplaneEnvironment(c Config, diagnosticsPort int32) []corev1.EnvVar { Value: "/host/dev", }, } + + // Ask the NVIDIA runtime for driver libraries only. The privileged + // container already sees /dev/nvidia*; no GPU resource is requested. + if c.nvidiaMode() != NVIDIAOff { + env = append(env, + corev1.EnvVar{Name: "NVIDIA_VISIBLE_DEVICES", Value: "all"}, + corev1.EnvVar{Name: "NVIDIA_DRIVER_CAPABILITIES", Value: "compute,utility"}, + ) + } + + return env } // DesiredDaemonSets builds steady-state placement, not a safe migration plan. diff --git a/internal/racer/workload/workload_test.go b/internal/racer/workload/workload_test.go new file mode 100644 index 000000000..455280406 --- /dev/null +++ b/internal/racer/workload/workload_test.go @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package workload + +import ( + "errors" + "testing" + + corev1 "k8s.io/api/core/v1" + + "github.com/Azure/unbounded/internal/racer/wire" +) + +func nvidiaLookup(mode string, set bool) func(string) (string, bool) { + values := map[string]string{ + "RACER_CLUSTER_ID": "00000000-0000-4000-8000-000000000001", + "RACER_CONTROL_URL": "https://racer-controller.unbounded-system.svc:8443", + "RACER_DATAPLANE_IMAGE": "racer-dataplane:test", + } + if set { + values["RACER_DATAPLANE_NVIDIA"] = mode + } + + return func(key string) (string, bool) { + value, ok := values[key] + return value, ok + } +} + +func envValue(c corev1.Container, name string) (string, bool) { + for _, e := range c.Env { + if e.Name == name { + return e.Value, true + } + } + + return "", false +} + +func TestNVIDIAInjectionModes(t *testing.T) { + for _, tc := range []struct { + name string + mode string + set bool + wantEnv bool + wantCDI bool + wantConfig NVIDIAInjection + }{ + {name: "default", wantEnv: true}, + {name: "off", mode: "off", set: true, wantConfig: NVIDIAOff}, + {name: "env", mode: "env", set: true, wantEnv: true, wantConfig: NVIDIAEnv}, + {name: "cdi", mode: "cdi", set: true, wantEnv: true, wantCDI: true, wantConfig: NVIDIACDI}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg, err := ConfigFromLookup(nvidiaLookup(tc.mode, tc.set)) + if err != nil { + t.Fatal(err) + } + + if cfg.NVIDIA != tc.wantConfig { + t.Fatalf("NVIDIA = %q, want %q", cfg.NVIDIA, tc.wantConfig) + } + + daemonSets, err := DesiredDaemonSets(cfg) + if err != nil { + t.Fatal(err) + } + + template := daemonSets[0].Spec.Template + container := template.Spec.Containers[0] + + devices, hasDevices := envValue(container, "NVIDIA_VISIBLE_DEVICES") + caps, hasCaps := envValue(container, "NVIDIA_DRIVER_CAPABILITIES") + + if hasDevices != tc.wantEnv || hasCaps != tc.wantEnv { + t.Fatalf("NVIDIA env present = %v/%v, want %v", hasDevices, hasCaps, tc.wantEnv) + } + + if tc.wantEnv && (devices != "all" || caps != "compute,utility") { + t.Fatalf("NVIDIA env = %q/%q", devices, caps) + } + + value, hasCDI := template.Annotations[CDIAnnotation] + if hasCDI != tc.wantCDI || (hasCDI && value != "nvidia.com/gpu=all") { + t.Fatalf("CDI annotation = %q (present %v), want present %v", value, hasCDI, tc.wantCDI) + } + + // GPU libraries must never come from a GPU resource request. + for _, list := range []corev1.ResourceList{container.Resources.Requests, container.Resources.Limits} { + if _, ok := list["nvidia.com/gpu"]; ok { + t.Fatal("dataplane must not request nvidia.com/gpu") + } + } + }) + } +} + +func TestNVIDIAInjectionRejectsUnknownMode(t *testing.T) { + for _, mode := range []string{"", "true", "auto", "CDI"} { + if _, err := ConfigFromLookup(nvidiaLookup(mode, true)); !errors.Is(err, wire.InvalidRequest) { + t.Fatalf("mode %q: err = %v, want InvalidRequest", mode, err) + } + } + + cfg, err := ConfigFromLookup(nvidiaLookup("", false)) + if err != nil { + t.Fatal(err) + } + + cfg.NVIDIA = "bogus" + if err := cfg.Validate(); !errors.Is(err, wire.InvalidRequest) { + t.Fatalf("Validate = %v, want InvalidRequest", err) + } + + // The zero value keeps the env default for programmatic callers. + cfg.NVIDIA = "" + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } +} From aa185e53336c007311b703c7fb07070aee1e722f Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 18:35:05 +0000 Subject: [PATCH 05/87] feat(racer): mount the client socket into labeled app containers --- deploy/racer/embed_test.go | 15 +- deploy/racer/mutating-webhook.yaml.tmpl | 38 ++- .../operator/components/racer/webhook_test.go | 13 +- internal/racer/injector/client_socket_test.go | 241 ++++++++++++++++++ internal/racer/injector/injector.go | 191 +++++++++++++- 5 files changed, 485 insertions(+), 13 deletions(-) create mode 100644 internal/racer/injector/client_socket_test.go diff --git a/deploy/racer/embed_test.go b/deploy/racer/embed_test.go index da3d1c274..b853542ac 100644 --- a/deploy/racer/embed_test.go +++ b/deploy/racer/embed_test.go @@ -27,7 +27,7 @@ func TestMutatingWebhookManifest(t *testing.T) { require.NoError(t, yaml.UnmarshalStrict(data, &config)) require.Equal(t, "MutatingWebhookConfiguration", config.Kind) require.Equal(t, "racer-object-sidecar-injector", config.Name) - require.Len(t, config.Webhooks, 1) + require.Len(t, config.Webhooks, 2) hook := config.Webhooks[0] require.Equal(t, "racer-object-sidecar.racer.unbounded-cloud.io", hook.Name) @@ -61,6 +61,19 @@ func TestMutatingWebhookManifest(t *testing.T) { require.Equal(t, admissionregistrationv1.NeverReinvocationPolicy, *hook.ReinvocationPolicy) require.Equal(t, int32(5), *hook.TimeoutSeconds) require.Equal(t, []string{"v1"}, hook.AdmissionReviewVersions) + + // The client-socket webhook matches its own label and skips pods that + // already carry the injected marker; everything else is identical. + socket := config.Webhooks[1] + require.Equal(t, "racer-client-socket.racer.unbounded-cloud.io", socket.Name) + require.Equal(t, map[string]string{injector.ClientSocketLabel: injector.EnabledValue}, socket.ObjectSelector.MatchLabels) + require.Empty(t, socket.ObjectSelector.MatchExpressions) + require.Len(t, socket.MatchConditions, 1) + require.Equal(t, "not-already-injected", socket.MatchConditions[0].Name) + require.Contains(t, socket.MatchConditions[0].Expression, "'"+injector.ClientSocketInjectedAnnotation+"'") + + socket.Name, socket.ObjectSelector, socket.MatchConditions = hook.Name, hook.ObjectSelector, hook.MatchConditions + require.Equal(t, hook, socket) } func TestControllerExposesWebhookPort(t *testing.T) { diff --git a/deploy/racer/mutating-webhook.yaml.tmpl b/deploy/racer/mutating-webhook.yaml.tmpl index 625e77c2a..32b4bc5e7 100644 --- a/deploy/racer/mutating-webhook.yaml.tmpl +++ b/deploy/racer/mutating-webhook.yaml.tmpl @@ -1,7 +1,11 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 # Injects the racer-object native sidecar into pods labeled -# racer.unbounded-cloud.io/object-sidecar=enabled. The operator fills caBundle. +# racer.unbounded-cloud.io/object-sidecar=enabled, and mounts the Racer client +# socket into pods labeled racer.unbounded-cloud.io/client-socket=enabled. +# Label selectors cannot express OR, so each label has its own webhook. Both +# call the same handler, which applies both opt-ins idempotently. +# The operator fills caBundle. apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: @@ -39,3 +43,35 @@ webhooks: sideEffects: None reinvocationPolicy: Never timeoutSeconds: 5 + - name: racer-client-socket.racer.unbounded-cloud.io + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: racer-controller + namespace: {{ .Namespace | default "unbounded-system" }} + path: /mutate-racer-object-sidecar + port: 9443 + rules: + - operations: ["CREATE"] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + scope: Namespaced + objectSelector: + matchLabels: + racer.unbounded-cloud.io/client-socket: enabled + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: [{{ .Namespace | default "unbounded-system" | quote }}] + matchConditions: + - name: not-already-injected + expression: >- + !has(object.metadata.annotations) || + !('racer.unbounded-cloud.io/client-socket-injected' in object.metadata.annotations) + failurePolicy: Fail + matchPolicy: Equivalent + sideEffects: None + reinvocationPolicy: Never + timeoutSeconds: 5 diff --git a/internal/operator/components/racer/webhook_test.go b/internal/operator/components/racer/webhook_test.go index 851b3c5d1..4480e9531 100644 --- a/internal/operator/components/racer/webhook_test.go +++ b/internal/operator/components/racer/webhook_test.go @@ -29,7 +29,18 @@ func TestWebhookInstalledWithExpectedShape(t *testing.T) { marker := &corev1.ConfigMap{} require.NoError(t, env.Client.Get(t.Context(), objectKey(env, markerName), marker)) require.NoError(t, validateRuntimeOwner(webhook, marker.UID)) - require.Len(t, webhook.Webhooks, 1) + require.Len(t, webhook.Webhooks, 2) + + // The client-socket webhook differs only in its label and match condition. + socketHook := webhook.Webhooks[1] + require.Equal(t, "racer-client-socket.racer.unbounded-cloud.io", socketHook.Name) + require.Equal(t, &metav1.LabelSelector{MatchLabels: map[string]string{"racer.unbounded-cloud.io/client-socket": "enabled"}}, socketHook.ObjectSelector) + require.Len(t, socketHook.MatchConditions, 1) + require.Contains(t, socketHook.MatchConditions[0].Expression, "racer.unbounded-cloud.io/client-socket-injected") + require.Equal(t, webhook.Webhooks[0].ClientConfig, socketHook.ClientConfig) + require.Equal(t, webhook.Webhooks[0].Rules, socketHook.Rules) + require.Equal(t, webhook.Webhooks[0].NamespaceSelector, socketHook.NamespaceSelector) + require.Equal(t, ptr.To(admissionv1.Fail), socketHook.FailurePolicy) hook := webhook.Webhooks[0] require.Equal(t, []admissionv1.RuleWithOperations{{ diff --git a/internal/racer/injector/client_socket_test.go b/internal/racer/injector/client_socket_test.go new file mode 100644 index 000000000..e71440d5c --- /dev/null +++ b/internal/racer/injector/client_socket_test.go @@ -0,0 +1,241 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package injector + +import ( + "reflect" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const testClientDir = "/run/racer/" + testCache + "/client" + +// newClientPod returns a pod opted into the client socket only. +func newClientPod(annotations map[string]string) *corev1.Pod { + a := map[string]string{ClientCacheAnnotation: testCache} + + for k, v := range annotations { + if v == "" { + delete(a, k) + continue + } + + a[k] = v + } + + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "ns", Labels: map[string]string{ClientSocketLabel: EnabledValue}, Annotations: a}, + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Name: "setup", Image: "app"}}, + Containers: []corev1.Container{{Name: "app", Image: "app"}, {Name: "helper", Image: "app"}}, + }, + } +} + +func socketMounted(c corev1.Container, volume string) bool { + mount := false + + for _, m := range c.VolumeMounts { + if reflect.DeepEqual(m, corev1.VolumeMount{Name: volume, MountPath: testClientDir, ReadOnly: true}) { + mount = true + } + } + + env := false + + for _, e := range c.Env { + if e.Name == SocketEnv && e.Value == testClientDir+"/socket" { + env = true + } + } + + return mount && env +} + +func TestMutateClientSocketAllContainers(t *testing.T) { + pod := newClientPod(nil) + if err := Mutate(pod, testCfg); err != nil { + t.Fatal(err) + } + + for _, c := range pod.Spec.Containers { + if !socketMounted(c, ClientVolumeName) { + t.Fatalf("container %s: %+v", c.Name, c) + } + } + + if len(pod.Spec.InitContainers) != 1 || len(pod.Spec.InitContainers[0].VolumeMounts) != 0 || len(pod.Spec.InitContainers[0].Env) != 0 { + t.Fatalf("init containers must be untouched and no sidecar added: %+v", pod.Spec.InitContainers) + } + + if len(pod.Spec.Volumes) != 1 { + t.Fatalf("volumes: %+v", pod.Spec.Volumes) + } + + v := pod.Spec.Volumes[0] + if v.Name != ClientVolumeName || v.HostPath.Path != testClientDir || *v.HostPath.Type != corev1.HostPathDirectory { + t.Fatalf("volume: %+v", v) + } + + if pod.Spec.NodeSelector[osLabel] != "linux" || pod.Spec.Affinity == nil { + t.Fatalf("placement: %+v %+v", pod.Spec.NodeSelector, pod.Spec.Affinity) + } + + if pod.Annotations[ClientSocketInjectedAnnotation] != testCache || pod.Annotations[InjectedAnnotation] != "" { + t.Fatalf("annotations: %+v", pod.Annotations) + } + + // A second admission pass must not change anything. + before := pod.DeepCopy() + if err := Mutate(pod, testCfg); err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(before, pod) { + t.Fatal("second Mutate changed the pod") + } +} + +func TestMutateClientSocketSelectedContainers(t *testing.T) { + pod := newClientPod(map[string]string{ClientContainersAnnotation: " helper "}) + if err := Mutate(pod, testCfg); err != nil { + t.Fatal(err) + } + + if socketMounted(pod.Spec.Containers[0], ClientVolumeName) || !socketMounted(pod.Spec.Containers[1], ClientVolumeName) { + t.Fatalf("containers: %+v", pod.Spec.Containers) + } +} + +func TestMutateClientSocketWithSidecarSharesVolume(t *testing.T) { + pod := newPod(nil) + pod.Labels[ClientSocketLabel] = EnabledValue + pod.Annotations[ClientCacheAnnotation] = testCache + + if err := Mutate(pod, testCfg); err != nil { + t.Fatal(err) + } + + if len(pod.Spec.Volumes) != 1 || pod.Spec.Volumes[0].Name != VolumeName { + t.Fatalf("want one shared volume, got %+v", pod.Spec.Volumes) + } + + if len(pod.Spec.InitContainers) != 1 || pod.Spec.InitContainers[0].Name != ContainerName { + t.Fatalf("sidecar missing: %+v", pod.Spec.InitContainers) + } + + if !socketMounted(pod.Spec.Containers[0], VolumeName) { + t.Fatalf("app container: %+v", pod.Spec.Containers[0]) + } + + // Distinct caches get distinct volumes. + pod = newPod(nil) + pod.Labels[ClientSocketLabel] = EnabledValue + pod.Annotations[ClientCacheAnnotation] = "other" + + if err := Mutate(pod, testCfg); err != nil { + t.Fatal(err) + } + + if len(pod.Spec.Volumes) != 2 || pod.Spec.Volumes[1].Name != ClientVolumeName || pod.Spec.Volumes[1].HostPath.Path != "/run/racer/other/client" { + t.Fatalf("volumes: %+v", pod.Spec.Volumes) + } +} + +func TestMutateClientSocketHostNetworkAllowed(t *testing.T) { + pod := newClientPod(nil) + pod.Spec.HostNetwork = true + + if err := Mutate(pod, testCfg); err != nil { + t.Fatal(err) + } +} + +func TestMutateClientSocketNoop(t *testing.T) { + for name, modify := range map[string]func(*corev1.Pod){ + "unlabeled": func(p *corev1.Pod) { p.Labels = nil; p.Annotations = nil }, + "label disabled": func(p *corev1.Pod) { p.Labels[ClientSocketLabel] = "disabled" }, + "already injected": func(p *corev1.Pod) { + p.Annotations = map[string]string{ClientSocketInjectedAnnotation: testCache} + }, + } { + t.Run(name, func(t *testing.T) { + pod := newClientPod(nil) + modify(pod) + before := pod.DeepCopy() + + if err := Mutate(pod, testCfg); err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(before, pod) { + t.Fatal("pod changed") + } + }) + } +} + +func TestMutateClientSocketErrors(t *testing.T) { + for _, tc := range []struct { + name string + anno map[string]string + modify func(*corev1.Pod) + want string + }{ + {name: "missing cache", anno: map[string]string{ClientCacheAnnotation: ""}, want: ClientCacheAnnotation}, + {name: "invalid cache", anno: map[string]string{ClientCacheAnnotation: "../x"}, want: ClientCacheAnnotation}, + {name: "unknown container", anno: map[string]string{ClientContainersAnnotation: "nope"}, want: ClientContainersAnnotation}, + {name: "init container", anno: map[string]string{ClientContainersAnnotation: "setup"}, want: ClientContainersAnnotation}, + {name: "duplicate container", anno: map[string]string{ClientContainersAnnotation: "app,app"}, want: ClientContainersAnnotation}, + {name: "empty list", anno: map[string]string{ClientContainersAnnotation: ""}, want: ClientContainersAnnotation}, + { + name: "volume name taken", + modify: func(p *corev1.Pod) { p.Spec.Volumes = []corev1.Volume{{Name: ClientVolumeName}} }, + want: ClientVolumeName, + }, + { + name: "mount path taken", + modify: func(p *corev1.Pod) { + p.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "x", MountPath: testClientDir}} + }, + want: "already mounts", + }, + { + name: "env taken", + modify: func(p *corev1.Pod) { p.Spec.Containers[1].Env = []corev1.EnvVar{{Name: SocketEnv, Value: "x"}} }, + want: SocketEnv, + }, + { + name: "windows os", + modify: func(p *corev1.Pod) { p.Spec.OS = &corev1.PodOS{Name: corev1.Windows} }, + want: "spec.os.name", + }, + { + name: "windows node selector", + modify: func(p *corev1.Pod) { p.Spec.NodeSelector = map[string]string{osLabel: "windows"} }, + want: osLabel, + }, + } { + t.Run(tc.name, func(t *testing.T) { + pod := newClientPod(tc.anno) + if tc.modify != nil { + tc.modify(pod) + } + + before := pod.DeepCopy() + + err := Mutate(pod, testCfg) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want mention of %q", err, tc.want) + } + + if !reflect.DeepEqual(before, pod) { + t.Fatal("pod changed on error") + } + }) + } +} diff --git a/internal/racer/injector/injector.go b/internal/racer/injector/injector.go index daf46229a..1f233be29 100644 --- a/internal/racer/injector/injector.go +++ b/internal/racer/injector/injector.go @@ -2,7 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 // Package injector implements a mutating admission handler that injects the -// racer-object sidecar into opted-in pods as a native (restartable init) sidecar. +// racer-object sidecar into opted-in pods as a native (restartable init) +// sidecar, and mounts the Racer client socket into opted-in app containers. package injector import ( @@ -45,6 +46,22 @@ const ( MemoryLimitAnnotation = "racer.unbounded-cloud.io/object-memory-limit" InjectedAnnotation = "racer.unbounded-cloud.io/object-sidecar-injected" + // ClientSocketLabel opts a pod into a read-only mount of the Racer client + // socket directory, independent of the racer-object sidecar. + ClientSocketLabel = "racer.unbounded-cloud.io/client-socket" + // ClientCacheAnnotation names the ClusterCache whose socket is mounted. + ClientCacheAnnotation = "racer.unbounded-cloud.io/client-cache" + // ClientContainersAnnotation optionally lists, comma-separated, the app + // containers that get the mount. The default is every app container. + ClientContainersAnnotation = "racer.unbounded-cloud.io/client-containers" + // ClientSocketInjectedAnnotation records the mounted cache. + ClientSocketInjectedAnnotation = "racer.unbounded-cloud.io/client-socket-injected" + // ClientVolumeName is the volume added when no volume already mounts the + // cache's client directory. + ClientVolumeName = "racer-client-socket" + // SocketEnv tells clients where the socket is. + SocketEnv = "RACER_SOCKET" + osLabel = "kubernetes.io/os" ) @@ -70,15 +87,26 @@ func (c Config) Validate() error { return nil } -// Mutate injects the racer-object sidecar into pod in place. Pods without the -// opt-in label, or that already contain a container named ContainerName, are -// left unchanged. Opted-in pods must set CacheAnnotation. Missing or invalid -// annotations return an error naming the annotation. +// Mutate applies both opt-ins to pod in place: the racer-object sidecar +// (EnabledLabel) and the client socket mount (ClientSocketLabel). Pods without +// either label are left unchanged. Missing or invalid annotations return an +// error naming the annotation. func Mutate(pod *corev1.Pod, cfg Config) error { if err := cfg.Validate(); err != nil { return err } + if err := mutateSidecar(pod, cfg); err != nil { + return err + } + + return mutateClientSocket(pod) +} + +// mutateSidecar injects the racer-object sidecar. Pods without the opt-in +// label, or that already contain a container named ContainerName, are left +// unchanged. Opted-in pods must set CacheAnnotation. +func mutateSidecar(pod *corev1.Pod, cfg Config) error { if pod.Labels[EnabledLabel] != EnabledValue { return nil } @@ -149,23 +177,166 @@ func Mutate(pod *corev1.Pod, cfg Config) error { }}, }) - if pod.Spec.NodeSelector == nil { - pod.Spec.NodeSelector = map[string]string{} + requireRacerNode(&pod.Spec) + + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + + pod.Annotations[InjectedAnnotation] = cfg.Image + + return nil +} + +// requireRacerNode places the pod on Linux nodes that run the dataplane. +func requireRacerNode(spec *corev1.PodSpec) { + if spec.NodeSelector == nil { + spec.NodeSelector = map[string]string{} + } + + spec.NodeSelector[osLabel] = "linux" + + addExclusionAffinity(spec) +} + +// mutateClientSocket mounts /run/racer//client read-only at the same +// path in the selected app containers and sets SocketEnv. A read-only mount +// still permits connect(2) on the socket: write permission on the socket +// inode is checked, not the mount's read-only flag. Pods without the label, +// or already marked with ClientSocketInjectedAnnotation, are left unchanged. +func mutateClientSocket(pod *corev1.Pod) error { + if pod.Labels[ClientSocketLabel] != EnabledValue { + return nil + } + + if _, done := pod.Annotations[ClientSocketInjectedAnnotation]; done { + return nil + } + + cache, ok := pod.Annotations[ClientCacheAnnotation] + if !ok { + return fmt.Errorf("annotation %s is required and must name the ClusterCache to bind to", ClientCacheAnnotation) + } + + if err := validateCache(cache); err != nil { + return fmt.Errorf("annotation %s has invalid cache name %q", ClientCacheAnnotation, cache) + } + + targets, err := clientTargets(pod) + if err != nil { + return err + } + + if pod.Spec.OS != nil && pod.Spec.OS.Name != corev1.Linux { + return fmt.Errorf("pod spec.os.name %q conflicts with the Racer client socket, which requires linux", pod.Spec.OS.Name) + } + + if v, ok := pod.Spec.NodeSelector[osLabel]; ok && v != "linux" { + return fmt.Errorf("pod nodeSelector %s=%q conflicts with the Racer client socket, which requires linux", osLabel, v) } - pod.Spec.NodeSelector[osLabel] = "linux" + clientDir := "/run/racer/" + cache + "/client" + socket := clientDir + "/socket" - addExclusionAffinity(&pod.Spec) + // Reuse a volume that already mounts this directory, such as the one the + // racer-object sidecar adds for the same cache. + volume := "" + + for _, v := range pod.Spec.Volumes { + if v.HostPath != nil && v.HostPath.Path == clientDir { + volume = v.Name + break + } + } + + if volume == "" { + for _, v := range pod.Spec.Volumes { + if v.Name == ClientVolumeName { + return fmt.Errorf("pod already defines a volume named %q", ClientVolumeName) + } + } + } + + for _, i := range targets { + c := &pod.Spec.Containers[i] + for _, m := range c.VolumeMounts { + if m.MountPath == clientDir { + return fmt.Errorf("container %q already mounts %s", c.Name, clientDir) + } + } + + for _, e := range c.Env { + if e.Name == SocketEnv { + return fmt.Errorf("container %q already sets %s", c.Name, SocketEnv) + } + } + } + + if volume == "" { + volume = ClientVolumeName + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: volume, + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ + Path: clientDir, + Type: ptr.To(corev1.HostPathDirectory), + }}, + }) + } + + for _, i := range targets { + c := &pod.Spec.Containers[i] + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{Name: volume, MountPath: clientDir, ReadOnly: true}) + c.Env = append(c.Env, corev1.EnvVar{Name: SocketEnv, Value: socket}) + } + + requireRacerNode(&pod.Spec) if pod.Annotations == nil { pod.Annotations = map[string]string{} } - pod.Annotations[InjectedAnnotation] = cfg.Image + pod.Annotations[ClientSocketInjectedAnnotation] = cache return nil } +// clientTargets returns indexes into pod.Spec.Containers. +func clientTargets(pod *corev1.Pod) ([]int, error) { + value, ok := pod.Annotations[ClientContainersAnnotation] + if !ok { + targets := make([]int, len(pod.Spec.Containers)) + for i := range targets { + targets[i] = i + } + + return targets, nil + } + + index := make(map[string]int, len(pod.Spec.Containers)) + for i, c := range pod.Spec.Containers { + index[c.Name] = i + } + + var targets []int + + seen := map[string]bool{} + + for _, name := range strings.Split(value, ",") { + name = strings.TrimSpace(name) + + i, found := index[name] + if !found || seen[name] { + return nil, fmt.Errorf("annotation %s must list distinct app container names, separated by commas", ClientContainersAnnotation) + } + + seen[name] = true + + targets = append(targets, i) + } + + return targets, nil +} + type sidecarSpec struct { cache string api string From d35c08c1137e8c09c558e16a6d9a394b142b7030 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 18:35:58 +0000 Subject: [PATCH 06/87] docs(racer): document GPU reads, client socket mount, and vLLM loading --- docs/content/guides/racer-vllm.md | 98 +++++++++++++++++++++++++++++++ docs/content/reference/racer.md | 59 +++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 docs/content/guides/racer-vllm.md diff --git a/docs/content/guides/racer-vllm.md b/docs/content/guides/racer-vllm.md new file mode 100644 index 000000000..de8957012 --- /dev/null +++ b/docs/content/guides/racer-vllm.md @@ -0,0 +1,98 @@ +--- +title: "Load vLLM Models from Racer" +weight: 11 +description: "Load model weights from a Racer cache straight into GPU memory with vLLM." +--- + +{{< callout type="warning" >}} +Racer is pre-release. Its APIs, flags, and on-disk formats may change without a +migration path. See the [Racer reference]({{< relref "reference/racer" >}}). +{{< /callout >}} + +This guide loads vLLM model weights from a Racer cache. The `racer` load format +reads each safetensors file through the local Racer dataplane. When GPU mode is +on, the dataplane writes the bytes straight into GPU memory. Otherwise the +client falls back to host memory, then to a plain stream. + +## Before You Start + +- A ClusterCache whose origin holds the model files. The + [Azure Blob guide]({{< relref "guides/azure-blob-cache" >}}) shows how to make + one. This guide uses a cache named `models`, a bucket named `llama`, and the + prefix `Llama-3.1-8B-Instruct`. +- NVIDIA nodes set up by the unbounded agent. + +## 1. Turn On GPU Mode + +Build the dataplane image with CUDA support and push it to the registry the +operator uses: + +```bash +podman build --build-arg RACER_NATIVE_CUDA=true \ + -t /racer-dataplane: -f images/racer-dataplane/Containerfile . +``` + +GPU mode is `auto` by default. To make the dataplane fail when it finds no GPU, +set it to `true`: + +```bash +kubectl -n unbounded-system patch configmap racer-dataplane-config \ + --type merge -p '{"data":{"RACER_ENABLE_GPU":"true"}}' +``` + +The dataplane gets the NVIDIA driver libraries without requesting a GPU. This +is on by default (`RACER_DATAPLANE_NVIDIA: env` in `racer-config`). See +[GPU reads]({{< relref "reference/racer" >}}#gpu-reads). + +## 2. Deploy vLLM + +Add the client socket label and the cache annotation to the vLLM pod: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: vllm + labels: + racer.unbounded-cloud.io/client-socket: enabled + annotations: + racer.unbounded-cloud.io/client-cache: models +spec: + containers: + - name: vllm + image: + command: + - vllm + - serve + - racer://models/llama/Llama-3.1-8B-Instruct + - --load-format + - racer + resources: + limits: + nvidia.com/gpu: 1 +``` + +The webhook mounts `/run/racer/models/client` read-only into the container and +sets `RACER_SOCKET` to the socket in it. To limit the mount to some containers, +add `racer.unbounded-cloud.io/client-containers: vllm`. + +## 3. Install racer-vllm + +Install both Python packages into the vLLM image: + +```bash +pip install ./clients/python/racer ./clients/python/racer-vllm +``` + +`racer-vllm` registers the `racer` load format with vLLM through a plugin entry +point. No other code change is needed. + +## 4. Run + +Pass `--load-format racer` and a model path of the form +`racer:////`. The prefix is the directory that holds +`model.safetensors.index.json` and the safetensors files. See the `racer-vllm` +README for how it finds the model config and tokenizer. + +The first load fills the cache from the origin. Later loads on any node read +from the cache. diff --git a/docs/content/reference/racer.md b/docs/content/reference/racer.md index e630b3c3a..6ee86e924 100644 --- a/docs/content/reference/racer.md +++ b/docs/content/reference/racer.md @@ -202,6 +202,8 @@ Numeric values are unsigned decimal strings: `268435456`, not `256Mi`. | `RACER_CRYPTO_HELPERS` | Unset (automatic) | Exact helper count, from 1 through the exact I/O count; requires `RACER_IO_WORKERS` | | `RACER_ALLOW_SMT` | `false` | Use eligible physical cores by default; `true` opts into logical CPUs | | `RACER_ENABLE_RDMA` | `auto` | `auto`, `true`, or `false`; see [modes](#rdma) | +| `RACER_ENABLE_GPU` | `auto` | `auto`, `true`, or `false`; see [GPU reads](#gpu-reads) | +| `RACER_GPU_STAGING_BYTES` | Unset (dataplane default) | Cap on ciphertext staging memory per GPU, in bytes | | `RACER_SHARES` | `4` | Enrollment proposal; explicit Node `shares` wins | | `RACER_PLAINTEXT_BYTES` / `RACER_CIPHERTEXT_BYTES` | Unset: `67108864` (64 MiB) / `134217728` (128 MiB) per final I/O worker | Independent plaintext/ciphertext admission budgets; an explicit value is node-wide and divided among I/O workers | | `RACER_DIRTY_BYTES` | Unset: `167772160` (160 MiB) per final I/O worker | Dirty-data budget; an explicit value is node-wide | @@ -677,6 +679,7 @@ also require matching Services, probes, network policy, and monitoring configura | `RACER_PEER_PORT` | `8082` | Managed dataplane peer listener and membership publication; 1024..65535 | | `RACER_DIAGNOSTICS_PORT` | Unset | Managed dataplane defaults to 9090, or 9091 if peer port is 9090; explicit port must be 1024..65535 and distinct | | `RACER_HOST_NETWORK` | `false` | Explicit Racer host-network opt-in; not implied by RDMA | +| `RACER_DATAPLANE_NVIDIA` | `env` | NVIDIA driver library injection for the dataplane: `off`, `env`, or `cdi`; see [GPU reads](#gpu-reads) | | `RACER_POD_NETWORK_NODES` | Unset | Not supported by operator-managed installations; the controller only enrolls Pods of DaemonSet `racer-dataplane`, so a nonempty value fails reconciliation | | `RACER_SNAPSHOT_MAX_AGE` | `30s` | Maximum replicated snapshot age | | `RACER_CERTIFICATE_LIFETIME` | `24h` | Dataplane leaf lifetime; 2 minutes..24 hours | @@ -933,6 +936,62 @@ for server limits, partial failures, and restart behavior. For tests, `github.com/Azure/unbounded/pkg/racersdk/racersdktest` runs a noncaching fake Racer. It does not test distributed or RDMA behavior. +## Client socket mount + +Pods that talk to the dataplane socket directly, such as the Python client, +opt in with a label. The admission webhook then mounts the cache's client +directory read-only and tells the app where the socket is. + +| Key | Meaning | +|-----|---------| +| Label `racer.unbounded-cloud.io/client-socket: enabled` | Turns on the mount | +| Annotation `racer.unbounded-cloud.io/client-cache` | ClusterCache name. Required | +| Annotation `racer.unbounded-cloud.io/client-containers` | Optional comma-separated app container names. Default: every app container. Init containers are never changed | +| Annotation `racer.unbounded-cloud.io/client-socket-injected` | Set by the webhook to the cache name; do not edit | + +Each selected container gets: + +- `/run/racer//client` mounted read-only at the same path. A read-only + mount still allows `connect` and `sendmsg` on the socket. +- `RACER_SOCKET=/run/racer//client/socket`. + +The pod is placed on Linux nodes without the `exclude` label, like sidecar pods. +The hostPath type is `Directory`, so the pod waits until the dataplane has +created the cache directory. Host networking is allowed. + +This label works alone or with `object-sidecar`. When both name the same cache, +the pod gets one shared volume. The webhook rejects the pod when the cache name +is missing or invalid, when a listed container does not exist, or when a +container already sets `RACER_SOCKET` or mounts that path. + +## GPU reads + +GPU reads let a client on the node read an object straight into its own NVIDIA +GPU memory. They need: + +- A dataplane image built with `--build-arg RACER_NATIVE_CUDA=true`. +- An NVIDIA node set up by the unbounded agent, which installs the NVIDIA + container runtime and a CDI spec. +- `RACER_ENABLE_GPU` in `racer-dataplane-config`. `auto` turns GPU mode off + when no usable GPU is found. `true` makes startup fail instead. `false` + turns it off. + +The dataplane does not request `nvidia.com/gpu`. It is privileged, so it +already sees `/dev/nvidia*`. It only needs the host driver libraries. +`RACER_DATAPLANE_NVIDIA` in `racer-config` controls how it asks for them: + +| `RACER_DATAPLANE_NVIDIA` | Behavior | +|--------------------------|----------| +| `env` (default) | Sets `NVIDIA_VISIBLE_DEVICES=all` and `NVIDIA_DRIVER_CAPABILITIES=compute,utility` on the dataplane container. The NVIDIA runtime injects the libraries. Other runtimes ignore these variables, so nodes without NVIDIA are not affected | +| `cdi` | Also adds the Pod annotation `cdi.k8s.io/racer-dataplane: nvidia.com/gpu=all`. containerd fails to create the container on any node that cannot resolve this CDI device, so use it only when every dataplane node has an NVIDIA CDI spec | +| `off` | Adds nothing | + +On nodes set up by the unbounded agent, the NVIDIA runtime is the default +containerd runtime and runs in CDI mode, so `env` is enough there. These env +entries are operator wiring: workload overrides cannot change them. + +See [Load vLLM models from Racer]({{< relref "guides/racer-vllm" >}}). + ## RDMA RDMA is optional per hop. Peers need the same **nonempty Site**, the page's selected rail From 3d9b68c8d92a6e5372f88611a07a6315298c6010 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 18:37:11 +0000 Subject: [PATCH 07/87] docs(racer): match vLLM guide to the racer-vllm fetch flow --- docs/content/guides/racer-vllm.md | 34 ++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/docs/content/guides/racer-vllm.md b/docs/content/guides/racer-vllm.md index de8957012..4c7f90645 100644 --- a/docs/content/guides/racer-vllm.md +++ b/docs/content/guides/racer-vllm.md @@ -62,19 +62,26 @@ spec: - name: vllm image: command: - - vllm - - serve - - racer://models/llama/Llama-3.1-8B-Instruct - - --load-format - - racer + - sh + - -c + - | + racer-vllm-fetch racer://models/llama/Llama-3.1-8B-Instruct /models/llama && + exec vllm serve /models/llama --load-format racer + volumeMounts: + - name: model-config + mountPath: /models resources: limits: nvidia.com/gpu: 1 + volumes: + - name: model-config + emptyDir: {} ``` The webhook mounts `/run/racer/models/client` read-only into the container and sets `RACER_SOCKET` to the socket in it. To limit the mount to some containers, -add `racer.unbounded-cloud.io/client-containers: vllm`. +add `racer.unbounded-cloud.io/client-containers: vllm`. The webhook does not +change init containers, so run `racer-vllm-fetch` in the vLLM container. ## 3. Install racer-vllm @@ -89,10 +96,17 @@ point. No other code change is needed. ## 4. Run -Pass `--load-format racer` and a model path of the form -`racer:////`. The prefix is the directory that holds -`model.safetensors.index.json` and the safetensors files. See the `racer-vllm` -README for how it finds the model config and tokenizer. +vLLM reads the model config and tokenizer before it loads weights, so loading +takes two steps: + +1. `racer-vllm-fetch racer://// ` copies + `config.json`, the tokenizer files, and the safetensors index into ``. + It also writes `racer-model.json`, which records the `racer://` location. + The prefix is the directory that holds `model.safetensors.index.json` and + the safetensors files. +2. `vllm serve --load-format racer` reads the weights through Racer. + +See the `racer-vllm` README for loader options such as `staging_bytes`. The first load fills the cache from the origin. Later loads on any node read from the cache. From 3ec5c32af44cf27a4b8ee2bbe5ed97fabd259630 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 18:37:43 +0000 Subject: [PATCH 08/87] docs(racer): align GPUDirect design with client fallback and NVIDIA knob --- designs/racer-gpudirect.md | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/designs/racer-gpudirect.md b/designs/racer-gpudirect.md index 163817311..b9ba5cd4a 100644 --- a/designs/racer-gpudirect.md +++ b/designs/racer-gpudirect.md @@ -117,8 +117,16 @@ Clients try modes in order: GPU, then host, then stream. `recv`. - A new dataplane without a usable GPU answers `gpu-read` with 501. -A client treats 400, 404, 405 and 501 on its first attempt at a mode as "mode -not supported" and remembers that for the socket path until the process exits. +A client remembers an unsupported mode for the socket path until the process +exits: + +- 501 marks the mode unsupported at once. +- 400, 404 and 405 can also mean a bad request, so the client falls back for + that read but marks the mode unsupported only when a later mode then + succeeds. + +A `host-read` refused because the destination is too small falls back to stream +for that read only. ### Security @@ -200,9 +208,13 @@ cuFile. v1 records are read through host memory. - The dataplane pod is privileged, so it already sees `/dev/nvidia*`. - CUDA needs the host driver's user-space libraries. The dataplane asks the NVIDIA runtime to inject them without requesting `nvidia.com/gpu`: - `NVIDIA_VISIBLE_DEVICES=all`, `NVIDIA_DRIVER_CAPABILITIES=compute,utility`, - and the CDI annotation for `nvidia.com/gpu=all`. This relies on the CDI setup - the unbounded agent does on NVIDIA nodes. On other nodes GPU mode stays off. + `NVIDIA_VISIBLE_DEVICES=all` and + `NVIDIA_DRIVER_CAPABILITIES=compute,utility`. The operator knob + `RACER_DATAPLANE_NVIDIA=off|env|cdi` (default `env`) controls this. `cdi` + also adds the CDI annotation for `nvidia.com/gpu=all`. It is opt-in because + containerd fails pod creation when a CDI device cannot be resolved, which + would break the DaemonSet on nodes without NVIDIA GPUs. On other nodes the + environment variables are ignored and GPU mode stays off. - The image gains a `RACER_NATIVE_CUDA` build argument that builds the shim and the `cuda` feature. - The injector gains a label that mounts the client socket directory into app From f2fd0b4c5d7c8c95445e03f07b245539e0dbf06d Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 18:25:20 +0000 Subject: [PATCH 09/87] feat(racer): expose physical extent translation on Slab --- cmd/racer-dataplane/alloc/src/slab.rs | 38 ++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/cmd/racer-dataplane/alloc/src/slab.rs b/cmd/racer-dataplane/alloc/src/slab.rs index fb0965b70..4f1d7145c 100644 --- a/cmd/racer-dataplane/alloc/src/slab.rs +++ b/cmd/racer-dataplane/alloc/src/slab.rs @@ -513,15 +513,25 @@ impl Slab { buffer: &AlignedBuffer, lease: &SegmentLease, ) -> Result<(Rc, Extent)> { - self.locate(extent, lease, |alignment, extent| { + self.locate_checked(extent, lease, |alignment, extent| { alignment.check(extent, buffer) }) } + /// Translate a leased logical extent to the worker-local descriptor and the + /// physical byte range on it, without submitting I/O or checking a buffer. + /// File backings return the logical extent unchanged; device backings add + /// the segment's placement offset. Callers that bypass io_uring (for + /// example cuFile) must keep the lease alive for the whole transfer, as + /// the leased generation is what keeps the bytes from being recycled. + pub fn locate(&self, extent: Extent, lease: &SegmentLease) -> Result<(Rc, Extent)> { + self.locate_checked(extent, lease, |_, _| Ok(())) + } + /// Validate a request's lease and extent, resolving its physical destination. /// `check` validates the transfer memory against both the logical and the /// physical extent with the slab's alignment. - fn locate( + fn locate_checked( &self, extent: Extent, lease: &SegmentLease, @@ -676,7 +686,7 @@ impl Slab { let (file, extent) = { let body = body.write_bytes()?; let header = header.as_slice(); - self.locate(extent, &lease, |alignment, extent| { + self.locate_checked(extent, &lease, |alignment, extent| { check_vectored(alignment, extent, header, body) })? }; @@ -1507,11 +1517,24 @@ mod tests { .unwrap(); let segments = Segments::new(4096); let _ = slab.open_configured(&segments).unwrap(); - drop(segments.append(4096).unwrap()); + let (first, first_extent) = segments.append(4096).unwrap(); let (lease, extent) = segments.append(4096).unwrap(); let buffer = slab.allocate(4096, ()).unwrap(); let (_, physical) = slab.submission(extent, &buffer, &lease).unwrap(); assert_eq!(physical, Extent::new(0, 4096).unwrap()); + // locate performs the same translation without a buffer. + let (descriptor, located) = slab.locate(extent, &lease).unwrap(); + assert_eq!(located, physical); + assert!(std::os::fd::AsRawFd::as_raw_fd(descriptor.as_ref()) >= 0); + assert_eq!( + slab.locate(first_extent, &first).unwrap().1, + Extent::new(8192, 4096).unwrap() + ); + drop(first); + assert!(matches!( + slab.locate(Extent::new(8192, 4096).unwrap(), &lease), + Err(Error::Corrupt) + )); assert!(matches!( slab.submission(Extent::new(0, 4096).unwrap(), &buffer, &lease), Err(Error::Corrupt) @@ -1546,6 +1569,13 @@ mod tests { }; placements[1].offset = offset; drop(opened); + // Translation itself does not enforce buffer alignment. + if offset == 1 { + assert_eq!( + slab.locate(extent, &lease).unwrap().1, + Extent::new(1, 4096).unwrap() + ); + } assert!( matches!(slab.submission(extent, &buffer, &lease), Err(error) if error == expected) ); From de531aab039b7d877098c719264796bb2584446f Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 19:16:56 +0000 Subject: [PATCH 10/87] feat(racer): parse record prefixes and locate aligned ciphertext Adds parse_prefix, DecodedRecord::direct_range, and direct_ciphertext on top of the upstream format version 2 record layout, so direct readers can validate a header through host memory and find the on-device ciphertext range. --- cmd/racer-dataplane/src/store.rs | 83 +++++++++++++++++++++++++- cmd/racer-dataplane/src/store/tests.rs | 36 +++++++++++ designs/racer-gpudirect.md | 23 ++++--- 3 files changed, 134 insertions(+), 8 deletions(-) diff --git a/cmd/racer-dataplane/src/store.rs b/cmd/racer-dataplane/src/store.rs index 7b2bacfcd..6d61fe74e 100644 --- a/cmd/racer-dataplane/src/store.rs +++ b/cmd/racer-dataplane/src/store.rs @@ -1351,6 +1351,10 @@ pub const MAX_RAW_HEADER_BYTES: usize = 16384; pub const MAX_HEADER_BYTES: usize = MAX_RAW_HEADER_BYTES + 4096; /// Smallest record unit; every stored `header_len` is a multiple of it. const MIN_RECORD_UNIT: usize = 512; +/// Ciphertext alignment that cuFile needs for direct-to-GPU reads. A record +/// whose ciphertext starts at a multiple of this on the device can be read +/// straight into GPU memory; others are read through host memory. +pub const CIPHERTEXT_ALIGNMENT: usize = 4096; /// Ciphertext body that can be written in place, without a staging copy. /// @@ -1428,6 +1432,67 @@ pub struct DecodedRecord { pub ciphertext: std::ops::Range, pub checksum: u64, } +impl DecodedRecord { + /// Ciphertext byte range relative to the record start, if the record's + /// header block places the ciphertext at a CIPHERTEXT_ALIGNMENT boundary + /// and so supports direct-to-GPU reads. + pub fn direct_range(&self) -> Option> { + (self.header.format_version == FORMAT_VERSION + && self.ciphertext.start.is_multiple_of(CIPHERTEXT_ALIGNMENT)) + .then_some(self.ciphertext.start as u64..self.ciphertext.end as u64) + } +} +/// Where a published record's ciphertext lives on its backing file or device. +/// Returned only when the header block is a multiple of CIPHERTEXT_ALIGNMENT. +/// The descriptor is the worker-local O_DIRECT +/// descriptor the slab reads through, suitable for registration with cuFile. +pub struct DirectCiphertext { + pub descriptor: Rc, + /// Absolute byte offset of the first ciphertext byte on `descriptor`. + pub offset: u64, + /// Ciphertext length in bytes, including the AEAD tag. + pub length: usize, + /// Whether `offset` is a multiple of CIPHERTEXT_ALIGNMENT. This holds for + /// every such record whose physical start is 4 KiB aligned. See + /// `direct_ciphertext` for when that is guaranteed. + pub aligned: bool, +} +/// Translate a parsed record into its absolute on-device ciphertext location. +/// +/// Returns Ok(None) when the record's header block is not a multiple of +/// CIPHERTEXT_ALIGNMENT; such records must be read through host memory. +/// `lease` must be the lease for `location` and must stay alive until any +/// direct transfer completes. Record extents and header blocks are multiples +/// of the record unit, so the ciphertext is 4 KiB aligned on disk whenever the +/// record unit is a multiple of 4 KiB and the segment's physical start is 4 KiB +/// aligned. `aligned` reports the actual result rather than assuming it. +pub fn direct_ciphertext( + slabs: &Slab>, + location: &catalog::RecordLocation, + lease: &SegmentLease, + record: &DecodedRecord, +) -> Result> { + if record.header.extent != location.extent + || lease.id() != location.segment + || lease.generation() != location.generation + { + return Err(Error::CorruptRecord); + } + let Some(range) = record.direct_range() else { + return Ok(None); + }; + let (descriptor, physical) = slabs.locate(location.extent, lease)?; + let offset = physical + .offset() + .checked_add(range.start) + .ok_or(Error::CorruptRecord)?; + Ok(Some(DirectCiphertext { + descriptor, + offset, + length: (range.end - range.start) as usize, + aligned: offset.is_multiple_of(CIPHERTEXT_ALIGNMENT as u64), + })) +} struct RecordLayout { /// Unpadded header length, including the digest. header_bytes: usize, @@ -1639,10 +1704,26 @@ pub fn parse( ) -> Result { parse_bytes(buffer.bytes()?, extent) } +/// Parse and validate a whole record of `extent`'s length. pub fn parse_bytes(bytes: &[u8], extent: Extent) -> Result { if bytes.len() != extent.length() { return Err(Error::CorruptRecord); } + parse_header(bytes, extent) +} +/// Parse and validate a record header from a prefix of the record at `extent`, +/// without touching the ciphertext. The prefix must cover the whole header +/// block; reading min(MAX_HEADER_BYTES, extent length) bytes always does. This +/// lets a direct-to-GPU reader validate framing through host memory and then +/// transfer only `DecodedRecord::ciphertext`. The ciphertext checksum is not +/// verified here. +pub fn parse_prefix(prefix: &[u8], extent: Extent) -> Result { + if prefix.len() > extent.length() { + return Err(Error::CorruptRecord); + } + parse_header(prefix, extent) +} +fn parse_header(bytes: &[u8], extent: Extent) -> Result { let mut r = Decoder(bytes); if r.take(8)? != MAGIC { return Err(Error::CorruptRecord); @@ -1725,7 +1806,7 @@ pub fn parse_bytes(bytes: &[u8], extent: Extent) -> Result { let logical_bytes = header_len .checked_add(ciphertext_length as usize) .ok_or(Error::CorruptRecord)?; - if logical_bytes > bytes.len() { + if logical_bytes > extent.length() { return Err(Error::CorruptRecord); } Ok(DecodedRecord { diff --git a/cmd/racer-dataplane/src/store/tests.rs b/cmd/racer-dataplane/src/store/tests.rs index 849d6fb6b..7840ce079 100644 --- a/cmd/racer-dataplane/src/store/tests.rs +++ b/cmd/racer-dataplane/src/store/tests.rs @@ -3942,6 +3942,42 @@ mod records { admission.reclaim_buffers(); } + #[test] + fn prefix_parse_and_direct_range_follow_header_block_alignment() { + let admission = admission(); + for unit in [512, 4096] { + let alignment = Alignment::new(unit, unit as u64, unit).unwrap(); + let page = page(&admission, 3, 0, "cache", "v1"); + let length = record_length(&page, alignment).unwrap(); + let encoded = encode( + &page, + Generation(7), + alignment, + buffer(&admission, alignment, length), + ) + .unwrap(); + let bytes = encoded.buffer.bytes().unwrap(); + let extent = encoded.header.extent; + let full = parse_bytes(bytes, extent).unwrap(); + let block = header_block_len(&page, alignment).unwrap(); + let prefix = parse_prefix(&bytes[..block], extent).unwrap(); + assert_eq!(prefix.ciphertext, full.ciphertext); + assert_eq!(prefix.header.metadata, full.header.metadata); + assert!(parse_prefix(&bytes[..block - 1], extent).is_err()); + let mut long = bytes.to_vec(); + long.push(0); + assert!(parse_prefix(&long, extent).is_err()); + // Only a 4 KiB header block is eligible for direct reads. + let direct = full.direct_range(); + if unit == 4096 { + assert_eq!(direct, Some(4096..4096 + 19)); + } else { + assert_eq!(direct, None); + } + } + admission.reclaim_buffers(); + } + #[test] fn malformed_frames_are_rejected_without_unbounded_allocations() { for len in [1, 16, 512] { diff --git a/designs/racer-gpudirect.md b/designs/racer-gpudirect.md index b9ba5cd4a..55f20d9b9 100644 --- a/designs/racer-gpudirect.md +++ b/designs/racer-gpudirect.md @@ -190,7 +190,7 @@ dataplane's GPU memory and are wiped after use. Ciphertext reaches GPU staging memory owned by the dataplane in one of three ways: - **Memory, origin or HTTP peer:** one host-to-GPU copy. -- **Local disk:** cuFile reads it straight from NVMe (layout v2 only, below). +- **Local disk:** cuFile reads it straight from NVMe (aligned records only, below). - **RDMA peer:** the NIC writes it into GPU memory registered with `ibv_reg_dmabuf_mr`. The dma-buf comes from `cuMemGetHandleForAddressRange`. @@ -198,10 +198,19 @@ Ciphertext reaches GPU staging memory owned by the dataplane in one of three way ### Disk layout v2 -v1 records put the ciphertext right after a variable-length header, so it is not -aligned. v2 pads the header so the ciphertext starts on a 4 KiB boundary inside -the record. The dataplane writes v2 and still reads v1. Only v2 records can use -cuFile. v1 records are read through host memory. +The store already writes format version 2 records (see +`cmd/racer-dataplane/src/store.rs`): the header and its SHA-256 digest, then zero +padding up to `header_len`, then the ciphertext, then zero padding up to the +extent. `header_len` and the ciphertext body are rounded up to the record unit, +which is the slab's direct-I/O extent unit. Readers treat other versions, including +the old unpadded v1 layout, as cold misses. + +cuFile needs the ciphertext to start on a 4 KiB boundary on the device. A record +is eligible when its `header_len` is a multiple of 4 KiB, and its device offset is +4 KiB aligned when the record unit is a multiple of 4 KiB and the segment's +physical start is 4 KiB aligned. The dataplane does not assume this. It computes +the absolute offset for each read and falls back to host memory if the offset is +not aligned. ## Deployment @@ -241,7 +250,7 @@ rule (`internal/racerobject/racerobject.go`). ## Testing - Rust simulation tests: descriptor handling, writes that would leave the - window, disconnect during a write, counted leaks, v1 and v2 records. + window, disconnect during a write, counted leaks, aligned and unaligned records. - A CPU test that compiles the GPU AES-GCM header as C and compares it with the `crypto` crate. - CI builds and loads the CUDA shim. @@ -252,5 +261,5 @@ rule (`internal/racerobject/racerobject.go`). 1. Descriptor passing, `host-read`, the Python client and the vLLM plugin. 2. GPU discovery, the `gpu` crate, GPU decryption and `gpu-read`. -3. Disk layout v2 and cuFile reads. +3. Aligned disk records and cuFile reads. 4. GPUDirect RDMA from peers. From 712121248cfb61a439da5e00fa9aabba532ffc15 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 22:06:09 +0000 Subject: [PATCH 11/87] feat(racer): floor the page record unit at 4 KiB for direct reads The store wrote header blocks and bodies in the slab's extent unit, so on 512-byte sector devices ciphertext was rarely 4 KiB aligned and cuFile could not read it. Raise the writer's record unit to 4 KiB when the extent unit is smaller. Readers still accept 512-byte header blocks, so existing records remain readable and fall back to host memory. --- cmd/racer-dataplane/src/store.rs | 48 ++++++++++++++++++++++---- cmd/racer-dataplane/src/store/tests.rs | 47 ++++++++++++++----------- designs/racer-gpudirect.md | 20 ++++++----- 3 files changed, 80 insertions(+), 35 deletions(-) diff --git a/cmd/racer-dataplane/src/store.rs b/cmd/racer-dataplane/src/store.rs index 6d61fe74e..972d882d7 100644 --- a/cmd/racer-dataplane/src/store.rs +++ b/cmd/racer-dataplane/src/store.rs @@ -1503,14 +1503,21 @@ struct RecordLayout { /// `header_block` plus the unpadded ciphertext length. logical_bytes: usize, } -/// Record unit: the extent granularity, so every in-record offset that is a -/// multiple of it is both offset- and length-aligned on disk. +/// Record unit: a multiple of the extent granularity, so every in-record +/// offset that is a multiple of it is both offset- and length-aligned on disk. +/// It is at least CIPHERTEXT_ALIGNMENT, so header blocks and record extents are +/// 4 KiB multiples and ciphertext stays eligible for direct-to-GPU reads on +/// devices with smaller sectors. fn record_unit(alignment: Alignment) -> Result { let unit = alignment.extent(0, 1)?.length(); if !unit.is_multiple_of(MIN_RECORD_UNIT) { return Err(Error::InvalidConfiguration); } - Ok(unit) + if CIPHERTEXT_ALIGNMENT.is_multiple_of(unit) { + Ok(CIPHERTEXT_ALIGNMENT) + } else { + Ok(unit) + } } fn align_up(len: usize, unit: usize) -> Result { len.checked_next_multiple_of(unit) @@ -1521,7 +1528,13 @@ fn layout( generation: Generation, alignment: Alignment, ) -> Result { - let unit = record_unit(alignment)?; + layout_with_unit(page, generation, record_unit(alignment)?) +} +fn layout_with_unit( + page: &CiphertextCopy, + generation: Generation, + unit: usize, +) -> Result { let raw = raw_layout(page, generation)?; let header_block = align_up(raw, unit)?; if header_block > MAX_HEADER_BYTES { @@ -1673,11 +1686,34 @@ pub fn encode_at( generation: Generation, alignment: Alignment, offset: u64, - mut buffer: AlignedBuffer>, + buffer: AlignedBuffer>, ) -> Result { let layout = layout(page, generation, alignment)?; + encode_layout(page, generation, alignment, offset, buffer, layout) +} +/// Encode with the extent unit as the record unit, the layout written before +/// the record unit had a 4 KiB floor. Readers must keep accepting it. +#[cfg(test)] +pub(crate) fn encode_unit_aligned_at( + page: &CiphertextCopy, + generation: Generation, + alignment: Alignment, + offset: u64, + buffer: AlignedBuffer>, +) -> Result { + let layout = layout_with_unit(page, generation, alignment.extent(0, 1)?.length())?; + encode_layout(page, generation, alignment, offset, buffer, layout) +} +fn encode_layout( + page: &CiphertextCopy, + generation: Generation, + alignment: Alignment, + offset: u64, + mut buffer: AlignedBuffer>, + layout: RecordLayout, +) -> Result { let logical_bytes = layout.logical_bytes; - let extent = alignment.extent(offset, logical_bytes)?; + let extent = alignment.extent(offset, layout.header_block + layout.body_bytes)?; if extent.length() != layout.header_block + layout.body_bytes { return Err(Error::InvalidConfiguration); } diff --git a/cmd/racer-dataplane/src/store/tests.rs b/cmd/racer-dataplane/src/store/tests.rs index 7840ce079..3eea50411 100644 --- a/cmd/racer-dataplane/src/store/tests.rs +++ b/cmd/racer-dataplane/src/store/tests.rs @@ -3515,7 +3515,7 @@ mod records { generation, alignment, 1, - buffer(&admission, alignment, 1024), + buffer(&admission, alignment, 8192), ) .err(); assert_eq!(actual, Some(Error::CorruptRecord)); @@ -3540,7 +3540,7 @@ mod records { &page, Generation(7), alignment, - buffer(&admission, alignment, 1024), + buffer(&admission, alignment, 8192), ) .unwrap(); assert_eq!( @@ -3562,7 +3562,7 @@ mod records { Generation(0), alignment, 1, - buffer(&admission, alignment, 1024) + buffer(&admission, alignment, 8192) ) .err(), Some(Error::CorruptRecord) @@ -3693,10 +3693,11 @@ mod records { let admission = admission(); let page = page(&admission, 3, 0, "cache", "v1"); let alignment = Alignment::new(512, 512, 512).unwrap(); - let encoded = encode( + let encoded = crate::store::encode_unit_aligned_at( &page, Generation(7), alignment, + 0, buffer(&admission, alignment, 1024), ) .unwrap(); @@ -3722,7 +3723,7 @@ mod records { &page, Generation(7), alignment, - buffer(&admission, alignment, 1024), + buffer(&admission, alignment, 8192), ) .unwrap(); let parsed = parse(&encoded.buffer, encoded.header.extent).unwrap(); @@ -3751,7 +3752,7 @@ mod records { &page, Generation(7), alignment, - buffer(&admission, alignment, 1024), + buffer(&admission, alignment, 8192), ) .unwrap(); let original = encoded.buffer.bytes().unwrap(); @@ -3799,19 +3800,20 @@ mod records { )); } - /// Encode `page(3, 0, "cache", etag)` at 512-byte alignment. + /// Encode `page(3, 0, "cache", etag)` with a 512-byte record unit, the + /// densest layout readers accept. fn encoded_v2( admission: &flow_control::Quotas, etag: &str, ) -> (Vec, Extent) { let page = page(admission, 3, 0, "cache", etag); let alignment = Alignment::new(512, 512, 512).unwrap(); - let length = record_length(&page, alignment).unwrap(); - let encoded = encode( + let encoded = crate::store::encode_unit_aligned_at( &page, Generation(7), alignment, - buffer(admission, alignment, length), + 0, + buffer(admission, alignment, 1024), ) .unwrap(); ( @@ -3833,11 +3835,13 @@ mod records { page.metadata.content_type = content_type .map(|v| crate::model::ContentType::parse(v.as_bytes()).unwrap()); let extra = content_type.map_or(0, str::len); + // The record unit has a 4 KiB floor for direct reads. + let record_unit = unit.max(crate::store::CIPHERTEXT_ALIGNMENT); let block = header_block_len(&page, alignment).unwrap(); - assert_eq!(block, (real + extra).next_multiple_of(unit)); - assert_eq!(body_len(&page, alignment).unwrap(), unit); + assert_eq!(block, (real + extra).next_multiple_of(record_unit)); + assert_eq!(body_len(&page, alignment).unwrap(), record_unit); let record = record_length(&page, alignment).unwrap(); - assert_eq!(record, block + unit); + assert_eq!(record, block + record_unit); let mut staging = buffer(&admission, alignment, record); staging.bytes_mut().unwrap().fill(0xa5); let encoded = @@ -3943,7 +3947,7 @@ mod records { } #[test] - fn prefix_parse_and_direct_range_follow_header_block_alignment() { + fn prefix_parse_and_direct_range_cover_new_and_unit_aligned_records() { let admission = admission(); for unit in [512, 4096] { let alignment = Alignment::new(unit, unit as u64, unit).unwrap(); @@ -3967,14 +3971,15 @@ mod records { let mut long = bytes.to_vec(); long.push(0); assert!(parse_prefix(&long, extent).is_err()); - // Only a 4 KiB header block is eligible for direct reads. - let direct = full.direct_range(); - if unit == 4096 { - assert_eq!(direct, Some(4096..4096 + 19)); - } else { - assert_eq!(direct, None); - } + // The 4 KiB record unit floor makes every new record eligible. + assert_eq!(full.direct_range(), Some(4096..4096 + 19)); } + // Records written with a 512-byte header block still parse, but must + // be read through host memory. + let (bytes, extent) = encoded_v2(&admission, "v1"); + let old = parse_bytes(&bytes, extent).unwrap(); + assert_eq!(old.ciphertext, 512..512 + 19); + assert_eq!(old.direct_range(), None); admission.reclaim_buffers(); } diff --git a/designs/racer-gpudirect.md b/designs/racer-gpudirect.md index 55f20d9b9..3459bed27 100644 --- a/designs/racer-gpudirect.md +++ b/designs/racer-gpudirect.md @@ -201,14 +201,18 @@ Ciphertext reaches GPU staging memory owned by the dataplane in one of three way The store already writes format version 2 records (see `cmd/racer-dataplane/src/store.rs`): the header and its SHA-256 digest, then zero padding up to `header_len`, then the ciphertext, then zero padding up to the -extent. `header_len` and the ciphertext body are rounded up to the record unit, -which is the slab's direct-I/O extent unit. Readers treat other versions, including -the old unpadded v1 layout, as cold misses. - -cuFile needs the ciphertext to start on a 4 KiB boundary on the device. A record -is eligible when its `header_len` is a multiple of 4 KiB, and its device offset is -4 KiB aligned when the record unit is a multiple of 4 KiB and the segment's -physical start is 4 KiB aligned. The dataplane does not assume this. It computes +extent. `header_len` and the ciphertext body are rounded up to the record unit. +Readers treat other versions, including the old unpadded v1 layout, as cold +misses. + +cuFile needs the ciphertext to start on a 4 KiB boundary on the device, so the +writer uses a record unit of at least 4 KiB: the slab's direct-I/O extent unit, +raised to 4 KiB when the extent unit is smaller. Readers still accept any +`header_len` that is a multiple of 512 bytes, so records written with a smaller +unit stay readable and are read through host memory. With 4 KiB record units, +every record in a segment starts 4 KiB aligned relative to the segment, and the +ciphertext is 4 KiB aligned on the device when the segment's physical start is +too. The dataplane does not assume this. It computes the absolute offset for each read and falls back to host memory if the offset is not aligned. From 78b8dfd4cdf1ebaf5ca9c311fa45a1dff10a2ddd Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 18:30:51 +0000 Subject: [PATCH 12/87] feat(racer): expose zeroizing XAES-256-GCM page subkey derivation --- cmd/racer-dataplane/crypto/src/lib.rs | 61 ++++++++++++++++- .../crypto/tests/primitives.rs | 68 +++++++++++++++++++ 2 files changed, 128 insertions(+), 1 deletion(-) diff --git a/cmd/racer-dataplane/crypto/src/lib.rs b/cmd/racer-dataplane/crypto/src/lib.rs index f38e3b726..cb9343894 100644 --- a/cmd/racer-dataplane/crypto/src/lib.rs +++ b/cmd/racer-dataplane/crypto/src/lib.rs @@ -153,7 +153,7 @@ impl PageCipher { } /// Derive the per-nonce AES-256-GCM key Kx from the first 12 nonce bytes. - fn derive(&self, nonce: &[u8; 24]) -> Result { + fn derive_key(&self, nonce: &[u8; 24]) -> Result, Error> { use aes::cipher::BlockCipherEncrypt; let mut kx = Zeroizing::new([0u8; 32]); for (index, half) in kx.chunks_exact_mut(16).enumerate() { @@ -166,6 +166,11 @@ impl PageCipher { self.aes .encrypt_block(half.try_into().map_err(|_| Error(()))?); } + Ok(kx) + } + + fn derive(&self, nonce: &[u8; 24]) -> Result { + let kx = self.derive_key(nonce)?; let key = ring::aead::UnboundKey::new(&ring::aead::AES_256_GCM, &kx[..]) .map_err(|_| Error(()))?; Ok(ring::aead::LessSafeKey::new(key)) @@ -320,6 +325,46 @@ where .map_err(|_| Error(())) } +/// The AES-256-GCM key and 96-bit nonce that XAES-256-GCM uses for one +/// 24-byte nonce. Lets a page be decrypted by a plain AES-256-GCM +/// implementation, such as the GPU kernels. The key is wiped on drop. +/// Intentionally does not implement Debug or Clone. +pub struct DerivedPageKey { + key: Zeroizing<[u8; 32]>, + + nonce: [u8; 12], +} + +impl DerivedPageKey { + /// The derived AES-256 key. Callers must not persist it. + pub fn key(&self) -> &[u8; 32] { + &self.key + } + + /// The 12-byte AES-GCM nonce: the last 12 bytes of the XAES nonce. + pub fn nonce(&self) -> &[u8; 12] { + &self.nonce + } +} + +/// Derive the AES-256-GCM key and nonce XAES-256-GCM uses for `nonce`, as +/// specified at : +/// `L = AES-256(K, 0^128)`, `K1 = L << 1` (xor `0x87` on carry), and the key is +/// `AES-256(K, [0,1,'X',0] || N[..12] ^ K1) || AES-256(K, [0,2,'X',0] || N[..12] ^ K1)`. +/// Shares the derivation [`seal`] and [`open`] use, so the result matches them +/// and `xaes_256_gcm::Xaes256Gcm` 0.1 byte for byte (see the tests). +pub fn xaes_derive(key: &[u8; 32], nonce: &[u8; 24]) -> DerivedPageKey { + let derived = PageCipher::new(key) + .derive_key(nonce) + .expect("16-byte halves always convert to AES blocks"); + let mut gcm_nonce = [0; 12]; + gcm_nonce.copy_from_slice(&nonce[12..]); + DerivedPageKey { + key: derived, + nonce: gcm_nonce, + } +} + /// Seal the unchanged XChaCha20-Poly1305 credential format, not the page format. /// The caller must never reuse a nonce with the same key. pub fn seal_credentials( @@ -1084,6 +1129,20 @@ pub mod identity { .map_err(|_| Error::CorruptRecord) } + /// Derive the per-page AES-256-GCM key and nonce for offloaded page + /// decryption (the GPU path). Checks cache, key ID and purpose like + /// [`Self::open_page`]. Never exposes the epoch key itself; the + /// derived key is valid for this one XAES nonce and wiped on drop. + pub fn derive_page_subkey( + &self, + cache: &CacheId, + id: KeyId, + nonce: &[u8; 24], + ) -> Result { + let material = self.bound_material(cache, id, KeyPurpose::Page)?; + Ok(crate::xaes_derive(material, nonce)) + } + /// Seal credentials using only a credential epoch. Caller owns nonce and AAD. pub fn seal_credentials( &self, diff --git a/cmd/racer-dataplane/crypto/tests/primitives.rs b/cmd/racer-dataplane/crypto/tests/primitives.rs index 46d110b36..437b4495c 100644 --- a/cmd/racer-dataplane/crypto/tests/primitives.rs +++ b/cmd/racer-dataplane/crypto/tests/primitives.rs @@ -918,6 +918,33 @@ mod identity_workflows { } } + /// Offload derivation equals the primitive and enforces the lease binding. + #[test] + fn derived_page_subkey_matches_primitive_and_enforces_binding() { + let (_, _, roots) = issued_with(|_| {}); + let keys = keyring(roots); + let cache = CacheId(CACHE.into()); + let page = keys.active(&cache, KeyPurpose::Page).unwrap(); + let credentials = keys.active(&cache, KeyPurpose::OriginCredentials).unwrap(); + let nonce = *b"abcdefghijklMNOPQRSTUVWX"; + let derived = page.derive_page_subkey(&cache, page.id(), &nonce).unwrap(); + let expected = racer_crypto::xaes_derive(&[7; 32], &nonce); + assert_eq!(derived.key(), expected.key()); + assert_eq!(derived.nonce(), b"MNOPQRSTUVWX"); + assert!(matches!( + page.derive_page_subkey(&cache, credentials.id(), &nonce), + Err(Error::MissingKey) + )); + assert!(matches!( + page.derive_page_subkey(&CacheId("wrong".into()), page.id(), &nonce), + Err(Error::MissingKey) + )); + assert!(matches!( + credentials.derive_page_subkey(&cache, credentials.id(), &nonce), + Err(Error::MissingKey) + )); + } + /// Page and credential operations enforce bindings without touching failed output. #[test] fn borrowed_outputs_enforce_purpose_cache_id_and_retained_epoch() { @@ -1348,3 +1375,44 @@ fn xaes_tamper_zeroes_output_and_length_errors_do_not() { assert!(seal(&key, &nonce, aad, &plaintext, &mut long).is_err()); assert!(long.iter().all(|byte| *byte == 0xa5)); } + +/// The XAES derivation lets plain AES-256-GCM open XAES-256-GCM pages, and it +/// matches the upstream implementation for nonce halves and key bytes. +#[test] +fn xaes_derivation_matches_upstream_xaes_256_gcm() { + use xaes_256_gcm::aes_gcm::{ + Aes256Gcm, KeyInit, + aead::{AeadInOut, inout::InOutBuf}, + }; + + let mut state = 0x9e37_79b9_7f4a_7c15u64; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state as u8 + }; + for round in 0..64 { + let key: [u8; 32] = std::array::from_fn(|_| next()); + let nonce: [u8; 24] = std::array::from_fn(|_| next()); + let plaintext: Vec = (0..round * 7).map(|_| next()).collect(); + let aad: Vec = (0..round % 40).map(|_| next()).collect(); + let mut sealed = vec![0; plaintext.len() + TAG_LEN]; + seal(&key, &nonce, &aad, &plaintext, &mut sealed).unwrap(); + + let derived = racer_crypto::xaes_derive(&key, &nonce); + assert_eq!(derived.nonce(), &nonce[12..]); + let cipher = Aes256Gcm::new(derived.key().into()); + let (ciphertext, tag) = sealed.split_at(plaintext.len()); + let mut out = vec![0; plaintext.len()]; + cipher + .decrypt_inout_detached( + derived.nonce().into(), + &aad, + InOutBuf::new(ciphertext, &mut out).unwrap(), + tag.try_into().unwrap(), + ) + .unwrap(); + assert_eq!(out, plaintext); + } +} From b1c794f1fd23675688299c9661f02198c8440cab Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 18:32:16 +0000 Subject: [PATCH 13/87] feat(racer): add racer-gpu crate with C AES-GCM and CRC64 decrypt core --- cmd/racer-dataplane/Cargo.lock | 10 + cmd/racer-dataplane/Cargo.toml | 2 +- cmd/racer-dataplane/gpu/Cargo.toml | 24 ++ cmd/racer-dataplane/gpu/build.rs | 17 ++ cmd/racer-dataplane/gpu/native/gcm_core.h | 339 ++++++++++++++++++++++ cmd/racer-dataplane/gpu/native/gcm_host.c | 70 +++++ cmd/racer-dataplane/gpu/src/gcm.rs | 154 ++++++++++ cmd/racer-dataplane/gpu/src/lib.rs | 2 + cmd/racer-dataplane/gpu/tests/gcm_core.rs | 186 ++++++++++++ 9 files changed, 803 insertions(+), 1 deletion(-) create mode 100644 cmd/racer-dataplane/gpu/Cargo.toml create mode 100644 cmd/racer-dataplane/gpu/build.rs create mode 100644 cmd/racer-dataplane/gpu/native/gcm_core.h create mode 100644 cmd/racer-dataplane/gpu/native/gcm_host.c create mode 100644 cmd/racer-dataplane/gpu/src/gcm.rs create mode 100644 cmd/racer-dataplane/gpu/src/lib.rs create mode 100644 cmd/racer-dataplane/gpu/tests/gcm_core.rs diff --git a/cmd/racer-dataplane/Cargo.lock b/cmd/racer-dataplane/Cargo.lock index f4f9f1beb..603b35cdd 100644 --- a/cmd/racer-dataplane/Cargo.lock +++ b/cmd/racer-dataplane/Cargo.lock @@ -885,6 +885,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "racer-gpu" +version = "0.1.0" +dependencies = [ + "cc", + "libc", + "racer-crypto", + "zeroize", +] + [[package]] name = "racer-object-wire" version = "0.1.0" diff --git a/cmd/racer-dataplane/Cargo.toml b/cmd/racer-dataplane/Cargo.toml index a21a68006..475cd8a79 100644 --- a/cmd/racer-dataplane/Cargo.toml +++ b/cmd/racer-dataplane/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [".", "runtime", "alloc", "crypto", "verbs", "http", "topology", "telemetry", "flow", "control-wire", "controlplane", "uds-endpoint", "wire-codec", "peer-wire", "object-wire"] +members = [".", "runtime", "alloc", "crypto", "verbs", "gpu", "http", "topology", "telemetry", "flow", "control-wire", "controlplane", "uds-endpoint", "wire-codec", "peer-wire", "object-wire"] resolver = "3" [package] diff --git a/cmd/racer-dataplane/gpu/Cargo.toml b/cmd/racer-dataplane/gpu/Cargo.toml new file mode 100644 index 000000000..12c8c78ab --- /dev/null +++ b/cmd/racer-dataplane/gpu/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "racer-gpu" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" +publish = false +description = "CUDA GPU discovery, client allocation import, staging, and GPU page decryption" + +[features] +default = [] +# Load libracer_gpu_cuda.so.1 (native/gpu_cuda.c) at runtime. +native = [] +# In-process fake GPUs backed by host memory, for tests without hardware. +simulation = [] + +[dependencies] +libc = "0.2" +zeroize = "1" + +[build-dependencies] +cc = "1" + +[dev-dependencies] +racer-crypto = { path = "../crypto" } diff --git a/cmd/racer-dataplane/gpu/build.rs b/cmd/racer-dataplane/gpu/build.rs new file mode 100644 index 000000000..44fd1d163 --- /dev/null +++ b/cmd/racer-dataplane/gpu/build.rs @@ -0,0 +1,17 @@ +//! Compile the host emulation of the GPU decrypt kernels (`native/gcm_host.c`, +//! which includes `native/gcm_core.h`) into a static library. Only the +//! simulation backend and tests reference it, so release builds without them +//! link none of it. `libracer_gpu_cuda.so.1` is built separately by +//! `images/racer-dataplane/build-native-cuda.sh`. +fn main() { + println!("cargo:rerun-if-changed=native/gcm_core.h"); + println!("cargo:rerun-if-changed=native/gcm_host.c"); + cc::Build::new() + .file("native/gcm_host.c") + .std("c99") + .opt_level(3) + .warnings(true) + .extra_warnings(true) + .warnings_into_errors(true) + .compile("racer_gpu_gcm_host"); +} diff --git a/cmd/racer-dataplane/gpu/native/gcm_core.h b/cmd/racer-dataplane/gpu/native/gcm_core.h new file mode 100644 index 000000000..a4ca5ee98 --- /dev/null +++ b/cmd/racer-dataplane/gpu/native/gcm_core.h @@ -0,0 +1,339 @@ +/* AES-256-GCM decrypt-with-verification and CRC-64/XZ, shared by the CUDA + * kernels (gcm_kernel.cu), the host shim (gpu_cuda.c), the simulation backend + * and the CPU tests (gcm_host.c). It compiles as C99 and as CUDA. + * + * Racer pages are sealed with XAES-256-GCM (https://c2sp.org/XAES-256-GCM). + * The CPU derives the per-page AES-256 key and 12-byte GCM nonce, so this file + * only implements plain AES-256-GCM (NIST SP 800-38D) on the derived values. + * The sealed page is ciphertext followed by the 16-byte tag; the stored + * checksum is CRC-64/XZ over the whole sealed page (racer_crypto::crc64 over + * CiphertextPage::bytes()). + * + * The work is split so it runs in parallel: + * + * gcm_prepare host: key schedule, H, E(K, J0), and the GHASH terms + * that do not depend on the ciphertext (AAD, lengths). + * gcm_chunk_partial one per GCM_CHUNK bytes: CRC of the chunk shifted to + * its final position, and the chunk's GHASH sum + * multiplied by the power of H it has in the full sum. + * gcm_finalize XOR of all partials (both sums are linear), then the + * constant-time tag compare and the CRC compare. + * gcm_ctr_block one per 16-byte block: CTR decryption. + * + * GHASH is sum(X_i * H^(m-i+1)) over m blocks, so chunk j with last block + * b_j contributes Horner(chunk) * H^(nc-b_j+1), where nc is the number of + * ciphertext blocks. CRC-64/XZ composes like zlib's crc32_combine: + * crc(A||B) = crc(A) * x^(8|B|) mod P xor crc(B), so each chunk is shifted by + * the number of bytes after it and all chunks are XORed. + * + * Timing: GF(2^128) multiplication is constant time (no secret-dependent + * branches or indexes). AES uses an S-box lookup with secret indexes, so it is + * not constant time with respect to cache or shared-memory bank timing. The + * kernels keep the S-box in shared memory, where only bank conflicts (not a + * cache hierarchy shared with other tenants) can leak timing, and the key is a + * per-page derived key. A bitsliced AES would remove this; see the README. + */ +#ifndef RACER_GCM_CORE_H +#define RACER_GCM_CORE_H + +#include +#include + +#ifdef __CUDACC__ +#define GCM_FN __host__ __device__ static inline +#else +#define GCM_FN static inline +#endif + +/* Bytes of sealed input each partial covers. A multiple of 16. */ +#define GCM_CHUNK 4096u +/* Reflected CRC-64/XZ (ECMA-182) polynomial. */ +#define GCM_CRC64_POLY 0xC96C5795D7870F42ull +/* Verdict values. Only GCM_VERDICT_PASS lets CTR decryption write output. */ +#define GCM_VERDICT_NONE 0u +#define GCM_VERDICT_PASS 0x600Du +#define GCM_VERDICT_CRC 2u +#define GCM_VERDICT_TAG 3u + +/* FIPS-197 S-box. */ +#define GCM_SBOX_INIT { \ + 0x63,0x7c,0x77,0x7b,0xf2,0x6b,0x6f,0xc5,0x30,0x01,0x67,0x2b,0xfe,0xd7,0xab,0x76, \ + 0xca,0x82,0xc9,0x7d,0xfa,0x59,0x47,0xf0,0xad,0xd4,0xa2,0xaf,0x9c,0xa4,0x72,0xc0, \ + 0xb7,0xfd,0x93,0x26,0x36,0x3f,0xf7,0xcc,0x34,0xa5,0xe5,0xf1,0x71,0xd8,0x31,0x15, \ + 0x04,0xc7,0x23,0xc3,0x18,0x96,0x05,0x9a,0x07,0x12,0x80,0xe2,0xeb,0x27,0xb2,0x75, \ + 0x09,0x83,0x2c,0x1a,0x1b,0x6e,0x5a,0xa0,0x52,0x3b,0xd6,0xb3,0x29,0xe3,0x2f,0x84, \ + 0x53,0xd1,0x00,0xed,0x20,0xfc,0xb1,0x5b,0x6a,0xcb,0xbe,0x39,0x4a,0x4c,0x58,0xcf, \ + 0xd0,0xef,0xaa,0xfb,0x43,0x4d,0x33,0x85,0x45,0xf9,0x02,0x7f,0x50,0x3c,0x9f,0xa8, \ + 0x51,0xa3,0x40,0x8f,0x92,0x9d,0x38,0xf5,0xbc,0xb6,0xda,0x21,0x10,0xff,0xf3,0xd2, \ + 0xcd,0x0c,0x13,0xec,0x5f,0x97,0x44,0x17,0xc4,0xa7,0x7e,0x3d,0x64,0x5d,0x19,0x73, \ + 0x60,0x81,0x4f,0xdc,0x22,0x2a,0x90,0x88,0x46,0xee,0xb8,0x14,0xde,0x5e,0x0b,0xdb, \ + 0xe0,0x32,0x3a,0x0a,0x49,0x06,0x24,0x5c,0xc2,0xd3,0xac,0x62,0x91,0x95,0xe4,0x79, \ + 0xe7,0xc8,0x37,0x6d,0x8d,0xd5,0x4e,0xa9,0x6c,0x56,0xf4,0xea,0x65,0x7a,0xae,0x08, \ + 0xba,0x78,0x25,0x2e,0x1c,0xa6,0xb4,0xc6,0xe8,0xdd,0x74,0x1f,0x4b,0xbd,0x8b,0x8a, \ + 0x70,0x3e,0xb5,0x66,0x48,0x03,0xf6,0x0e,0x61,0x35,0x57,0xb9,0x86,0xc1,0x1d,0x9e, \ + 0xe1,0xf8,0x98,0x11,0x69,0xd9,0x8e,0x94,0x9b,0x1e,0x87,0xe9,0xce,0x55,0x28,0xdf, \ + 0x8c,0xa1,0x89,0x0d,0xbf,0xe6,0x42,0x68,0x41,0x99,0x2d,0x0f,0xb0,0x54,0xbb,0x16 } + +/* Everything the device needs for one page. Wiped after use. */ +struct gcm_key { + uint8_t rk[240]; /* AES-256 round keys */ + uint8_t h[16]; /* E(K, 0^128) */ + uint8_t ekj0[16]; /* E(K, J0), the tag mask */ + uint8_t prefix[16]; /* GHASH terms for AAD and the length block */ + uint8_t nonce[12]; + uint32_t reserved; +}; + +/* One chunk's contribution. */ +struct gcm_partial { + uint8_t ghash[16]; + uint64_t crc; + uint64_t reserved; +}; + +/* Result of verification. Same layout as struct racer_gpu_verdict. */ +struct gcm_verdict { + uint32_t status; + uint32_t reserved; + uint64_t crc; +}; + +/* GF(2^128) element in GCM bit order, as two big-endian halves. */ +struct gcm_block { + uint64_t hi, lo; +}; + +GCM_FN uint64_t gcm_load64(const uint8_t *p) { + uint64_t v = 0; + for (int i = 0; i < 8; ++i) v = (v << 8) | p[i]; + return v; +} + +GCM_FN void gcm_store64(uint8_t *p, uint64_t v) { + for (int i = 7; i >= 0; --i) { p[i] = (uint8_t)v; v >>= 8; } +} + +GCM_FN struct gcm_block gcm_load(const uint8_t *p) { + struct gcm_block b; + b.hi = gcm_load64(p); + b.lo = gcm_load64(p + 8); + return b; +} + +/* Load up to 16 bytes, zero padding the rest. */ +GCM_FN struct gcm_block gcm_load_partial(const uint8_t *p, uint64_t n) { + uint8_t tmp[16]; + for (unsigned i = 0; i < 16; ++i) tmp[i] = i < n ? p[i] : 0; + return gcm_load(tmp); +} + +GCM_FN void gcm_store(uint8_t *p, struct gcm_block b) { + gcm_store64(p, b.hi); + gcm_store64(p + 8, b.lo); +} + +/* X * Y in GF(2^128), constant time (SP 800-38D algorithm 1). */ +GCM_FN struct gcm_block gcm_mul(struct gcm_block x, struct gcm_block y) { + struct gcm_block z = {0, 0}, v = y; + for (int i = 0; i < 128; ++i) { + uint64_t bit = i < 64 ? (x.hi >> (63 - i)) & 1 : (x.lo >> (127 - i)) & 1; + uint64_t mask = (uint64_t)0 - bit; + z.hi ^= v.hi & mask; + z.lo ^= v.lo & mask; + uint64_t lsb = (uint64_t)0 - (v.lo & 1); + v.lo = (v.lo >> 1) | (v.hi << 63); + v.hi = (v.hi >> 1) ^ (0xE100000000000000ull & lsb); + } + return z; +} + +/* H^k. k is public; the multiplications are constant time in H. */ +GCM_FN struct gcm_block gcm_pow(struct gcm_block h, uint64_t k) { + struct gcm_block r = {0x8000000000000000ull, 0}; /* multiplicative one */ + while (k) { + if (k & 1) r = gcm_mul(r, h); + h = gcm_mul(h, h); + k >>= 1; + } + return r; +} + +GCM_FN uint8_t gcm_xtime(uint8_t x) { + return (uint8_t)((x << 1) ^ (0x1b & (uint8_t)(0u - (x >> 7)))); +} + +/* FIPS-197 AES-256 key expansion into 15 round keys. */ +GCM_FN void gcm_aes256_expand(const uint8_t *sbox, const uint8_t key[32], uint8_t rk[240]) { + uint8_t rcon = 1; + for (int i = 0; i < 32; ++i) rk[i] = key[i]; + for (int i = 8; i < 60; ++i) { + uint8_t t[4]; + for (int j = 0; j < 4; ++j) t[j] = rk[4 * (i - 1) + j]; + if (i % 8 == 0) { + uint8_t first = t[0]; + t[0] = (uint8_t)(sbox[t[1]] ^ rcon); + t[1] = sbox[t[2]]; + t[2] = sbox[t[3]]; + t[3] = sbox[first]; + rcon = gcm_xtime(rcon); + } else if (i % 8 == 4) { + for (int j = 0; j < 4; ++j) t[j] = sbox[t[j]]; + } + for (int j = 0; j < 4; ++j) rk[4 * i + j] = (uint8_t)(rk[4 * (i - 8) + j] ^ t[j]); + } +} + +/* Encrypt one block with AES-256 round keys `rk`. */ +GCM_FN void gcm_aes256_encrypt(const uint8_t *sbox, const uint8_t *rk, const uint8_t in[16], + uint8_t out[16]) { + uint8_t s[16], t[16]; + for (int i = 0; i < 16; ++i) s[i] = (uint8_t)(in[i] ^ rk[i]); + for (int round = 1; round <= 14; ++round) { + /* SubBytes and ShiftRows. Byte r + 4c is row r, column c. */ + for (int c = 0; c < 4; ++c) + for (int r = 0; r < 4; ++r) t[r + 4 * c] = sbox[s[r + 4 * ((c + r) & 3)]]; + if (round != 14) { + for (int c = 0; c < 4; ++c) { + uint8_t a0 = t[4 * c], a1 = t[4 * c + 1], a2 = t[4 * c + 2], a3 = t[4 * c + 3]; + uint8_t all = (uint8_t)(a0 ^ a1 ^ a2 ^ a3); + s[4 * c] = (uint8_t)(a0 ^ all ^ gcm_xtime((uint8_t)(a0 ^ a1))); + s[4 * c + 1] = (uint8_t)(a1 ^ all ^ gcm_xtime((uint8_t)(a1 ^ a2))); + s[4 * c + 2] = (uint8_t)(a2 ^ all ^ gcm_xtime((uint8_t)(a2 ^ a3))); + s[4 * c + 3] = (uint8_t)(a3 ^ all ^ gcm_xtime((uint8_t)(a3 ^ a0))); + } + } else { + for (int i = 0; i < 16; ++i) s[i] = t[i]; + } + for (int i = 0; i < 16; ++i) s[i] ^= rk[16 * round + i]; + } + for (int i = 0; i < 16; ++i) out[i] = s[i]; +} + +/* Reflected CRC-64/XZ byte table. */ +GCM_FN void gcm_crc64_table(uint64_t table[256]) { + for (unsigned i = 0; i < 256; ++i) { + uint64_t c = i; + for (int k = 0; k < 8; ++k) c = (c & 1) ? (c >> 1) ^ GCM_CRC64_POLY : c >> 1; + table[i] = c; + } +} + +/* Complete CRC-64/XZ (init and xorout all ones) of `n` bytes. */ +GCM_FN uint64_t gcm_crc64(const uint64_t *table, const uint8_t *p, uint64_t n) { + uint64_t c = ~(uint64_t)0; + for (uint64_t i = 0; i < n; ++i) c = table[(c ^ p[i]) & 0xff] ^ (c >> 8); + return ~c; +} + +/* a * b mod P in the reflected representation (bit 63 is x^0). */ +GCM_FN uint64_t gcm_crc_mulmod(uint64_t a, uint64_t b) { + uint64_t m = 1ull << 63, p = 0; + if (!a) return 0; + for (;;) { + if (a & m) { + p ^= b; + if ((a & (m - 1)) == 0) break; + } + m >>= 1; + b = (b & 1) ? (b >> 1) ^ GCM_CRC64_POLY : b >> 1; + } + return p; +} + +/* crc * x^(8n) mod P: moves a complete CRC past n more bytes. */ +GCM_FN uint64_t gcm_crc_shift(uint64_t crc, uint64_t n) { + uint64_t p = 1ull << 63, base = 1ull << (63 - 8); + while (n) { + if (n & 1) p = gcm_crc_mulmod(base, p); + base = gcm_crc_mulmod(base, base); + n >>= 1; + } + return gcm_crc_mulmod(p, crc); +} + +/* Number of partials for a page of `n` plaintext bytes. */ +GCM_FN uint64_t gcm_chunks(uint64_t n) { + return (n + 16 + GCM_CHUNK - 1) / GCM_CHUNK; +} + +/* Host side: derive everything the kernels need from the derived key, + * nonce, AAD and plaintext length. */ +GCM_FN void gcm_prepare(const uint8_t *sbox, const uint8_t key[32], const uint8_t nonce[12], + const uint8_t *aad, uint64_t aad_len, uint64_t n, struct gcm_key *out) { + uint8_t block[16]; + gcm_aes256_expand(sbox, key, out->rk); + for (int i = 0; i < 16; ++i) block[i] = 0; + gcm_aes256_encrypt(sbox, out->rk, block, out->h); + for (int i = 0; i < 12; ++i) block[i] = out->nonce[i] = nonce[i]; + block[12] = 0; block[13] = 0; block[14] = 0; block[15] = 1; + gcm_aes256_encrypt(sbox, out->rk, block, out->ekj0); + out->reserved = 0; + + struct gcm_block h = gcm_load(out->h), y = {0, 0}; + for (uint64_t i = 0; i < aad_len; i += 16) { + struct gcm_block x = gcm_load_partial(aad + i, aad_len - i); + y.hi ^= x.hi; y.lo ^= x.lo; + y = gcm_mul(y, h); + } + uint64_t nc = (n + 15) / 16; + struct gcm_block prefix = gcm_mul(y, gcm_pow(h, nc + 1)); + struct gcm_block lengths = {aad_len * 8, n * 8}; + struct gcm_block lh = gcm_mul(lengths, h); + prefix.hi ^= lh.hi; prefix.lo ^= lh.lo; + gcm_store(out->prefix, prefix); + for (int i = 0; i < 16; ++i) block[i] = 0; +} + +/* Partial `j` over the sealed page (`n` ciphertext bytes then a 16-byte tag). */ +GCM_FN void gcm_chunk_partial(const uint64_t *crc_table, const struct gcm_key *key, + const uint8_t *sealed, uint64_t n, uint64_t j, + struct gcm_partial *out) { + uint64_t total = n + 16; + uint64_t start = j * GCM_CHUNK; + uint64_t end = start + GCM_CHUNK < total ? start + GCM_CHUNK : total; + out->crc = gcm_crc_shift(gcm_crc64(crc_table, sealed + start, end - start), total - end); + out->reserved = 0; + + struct gcm_block y = {0, 0}; + uint64_t gend = end < n ? end : n; + if (start < gend) { + struct gcm_block h = gcm_load(key->h); + for (uint64_t i = start; i < gend; i += 16) { + struct gcm_block x = gcm_load_partial(sealed + i, gend - i); + y.hi ^= x.hi; y.lo ^= x.lo; + y = gcm_mul(y, h); + } + uint64_t nc = (n + 15) / 16, last = (gend + 15) / 16; + y = gcm_mul(y, gcm_pow(h, nc - last + 1)); + } + gcm_store(out->ghash, y); +} + +/* Combine reduced partials, check the tag in constant time and the CRC. */ +GCM_FN void gcm_finalize(const struct gcm_key *key, const uint8_t *sealed, uint64_t n, + const uint8_t ghash[16], uint64_t crc, uint32_t check_crc, + uint64_t expected_crc, struct gcm_verdict *out) { + uint8_t diff = 0; + for (int i = 0; i < 16; ++i) + diff |= (uint8_t)(key->prefix[i] ^ ghash[i] ^ key->ekj0[i] ^ sealed[n + i]); + int crc_ok = !check_crc || crc == expected_crc; + out->crc = crc; + out->reserved = 0; + out->status = !crc_ok ? GCM_VERDICT_CRC : diff ? GCM_VERDICT_TAG : GCM_VERDICT_PASS; +} + +/* CTR-decrypt block `i` into `out`. Call only after a passing verdict. */ +GCM_FN void gcm_ctr_block(const uint8_t *sbox, const struct gcm_key *key, const uint8_t *sealed, + uint64_t n, uint64_t i, uint8_t *out) { + uint8_t counter[16], stream[16]; + uint32_t c = (uint32_t)(i + 2); + for (int k = 0; k < 12; ++k) counter[k] = key->nonce[k]; + counter[12] = (uint8_t)(c >> 24); counter[13] = (uint8_t)(c >> 16); + counter[14] = (uint8_t)(c >> 8); counter[15] = (uint8_t)c; + gcm_aes256_encrypt(sbox, key->rk, counter, stream); + uint64_t base = i * 16, len = n - base < 16 ? n - base : 16; + for (uint64_t k = 0; k < len; ++k) out[base + k] = (uint8_t)(sealed[base + k] ^ stream[k]); + for (int k = 0; k < 16; ++k) stream[k] = 0; +} + +#endif diff --git a/cmd/racer-dataplane/gpu/native/gcm_host.c b/cmd/racer-dataplane/gpu/native/gcm_host.c new file mode 100644 index 000000000..e1064e5cc --- /dev/null +++ b/cmd/racer-dataplane/gpu/native/gcm_host.c @@ -0,0 +1,70 @@ +/* Serial host emulation of the GPU decrypt kernels. + * + * Runs the same gcm_core.h steps in the same order the kernels do: prepare, + * one partial per chunk, XOR reduction, finalize, then CTR only on a passing + * verdict. The CPU tests compare it with the racer-crypto crate, and the + * simulation backend uses it as its "kernel". Linked statically into the Rust + * crate by build.rs; never part of libracer_gpu_cuda.so.1. */ +#include "gcm_core.h" + +#include +#include + +static const uint8_t host_sbox[256] = GCM_SBOX_INIT; + +/* Wipe memory the compiler may not elide. */ +static void wipe(void *p, size_t n) { + volatile uint8_t *v = (volatile uint8_t *)p; + while (n--) *v++ = 0; +} + +/* CRC-64/XZ through the chunked combine path. */ +uint64_t racer_gcm_host_crc64(const uint8_t *data, uint64_t length) { + uint64_t table[256], crc = 0; + gcm_crc64_table(table); + if (!length) return gcm_crc64(table, data, 0); + for (uint64_t start = 0; start < length; start += GCM_CHUNK) { + uint64_t end = start + GCM_CHUNK < length ? start + GCM_CHUNK : length; + crc ^= gcm_crc_shift(gcm_crc64(table, data + start, end - start), length - end); + } + return crc; +} + +/* One AES-256 block, for known-answer tests. */ +void racer_gcm_host_aes256(const uint8_t key[32], const uint8_t in[16], uint8_t out[16]) { + uint8_t rk[240]; + gcm_aes256_expand(host_sbox, key, rk); + gcm_aes256_encrypt(host_sbox, rk, in, out); + wipe(rk, sizeof(rk)); +} + +/* Verify and decrypt `sealed` (n ciphertext bytes then a 16-byte tag) into + * `out` (n bytes). `out` is written only when the verdict passes. + * Returns 0, or -1 if scratch memory cannot be allocated. */ +int racer_gcm_host_decrypt(const uint8_t key[32], const uint8_t nonce[12], const uint8_t *aad, + uint64_t aad_len, const uint8_t *sealed, uint64_t n, + uint32_t check_crc, uint64_t expected_crc, uint8_t *out, + struct gcm_verdict *verdict) { + struct gcm_key k; + uint64_t table[256]; + uint64_t chunks = gcm_chunks(n); + struct gcm_partial *partials = calloc(chunks, sizeof(*partials)); + verdict->status = GCM_VERDICT_NONE; + if (!partials) return -1; + gcm_crc64_table(table); + gcm_prepare(host_sbox, key, nonce, aad, aad_len, n, &k); + for (uint64_t j = 0; j < chunks; ++j) gcm_chunk_partial(table, &k, sealed, n, j, &partials[j]); + uint8_t ghash[16] = {0}; + uint64_t crc = 0; + for (uint64_t j = 0; j < chunks; ++j) { + for (int i = 0; i < 16; ++i) ghash[i] ^= partials[j].ghash[i]; + crc ^= partials[j].crc; + } + gcm_finalize(&k, sealed, n, ghash, crc, check_crc, expected_crc, verdict); + if (verdict->status == GCM_VERDICT_PASS) + for (uint64_t i = 0; i < (n + 15) / 16; ++i) gcm_ctr_block(host_sbox, &k, sealed, n, i, out); + wipe(&k, sizeof(k)); + wipe(partials, chunks * sizeof(*partials)); + free(partials); + return 0; +} diff --git a/cmd/racer-dataplane/gpu/src/gcm.rs b/cmd/racer-dataplane/gpu/src/gcm.rs new file mode 100644 index 000000000..35d429a61 --- /dev/null +++ b/cmd/racer-dataplane/gpu/src/gcm.rs @@ -0,0 +1,154 @@ +//! Host emulation of the GPU decrypt kernels, from `native/gcm_host.c`. +//! +//! It runs the exact `native/gcm_core.h` steps the CUDA kernels run, serially. +//! The simulation backend uses it as its kernel, and the tests compare it with +//! `racer-crypto`. + +use std::ffi::c_int; + +/// Verification result. Matches `struct gcm_verdict` and +/// `struct racer_gpu_verdict` in C. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct RawVerdict { + /// One of the `VERDICT_*` values. + pub status: u32, + + /// Always zero. + pub reserved: u32, + + /// CRC-64/XZ of the sealed page, computed by the kernel. + pub crc: u64, +} + +/// Verification did not run. +pub const VERDICT_NONE: u32 = 0; + +/// CRC (if requested) and tag matched; the plaintext was written. +pub const VERDICT_PASS: u32 = 0x600d; + +/// The CRC did not match. Nothing was written. +pub const VERDICT_CRC: u32 = 2; + +/// The tag did not match. Nothing was written. +pub const VERDICT_TAG: u32 = 3; + +/// Bytes covered by each parallel partial (`GCM_CHUNK`). +pub const CHUNK: u64 = 4096; + +unsafe extern "C" { + fn racer_gcm_host_crc64(data: *const u8, length: u64) -> u64; + + fn racer_gcm_host_aes256(key: *const [u8; 32], input: *const [u8; 16], out: *mut [u8; 16]); + + fn racer_gcm_host_decrypt( + key: *const [u8; 32], + nonce: *const [u8; 12], + aad: *const u8, + aad_len: u64, + sealed: *const u8, + n: u64, + check_crc: u32, + expected_crc: u64, + out: *mut u8, + verdict: *mut RawVerdict, + ) -> c_int; +} + +/// CRC-64/XZ through the chunked combine path the kernels use. +pub fn crc64(data: &[u8]) -> u64 { + unsafe { racer_gcm_host_crc64(data.as_ptr(), data.len() as u64) } +} + +/// One AES-256 block encryption. +pub fn aes256(key: &[u8; 32], block: &[u8; 16]) -> [u8; 16] { + let mut out = [0; 16]; + unsafe { racer_gcm_host_aes256(key, block, &mut out) }; + out +} + +/// Verify `sealed` (ciphertext then 16-byte tag) and decrypt it into `out` +/// with the derived AES-256-GCM key and nonce. `out` must be exactly the +/// ciphertext length and is written only when the verdict passes. +/// +/// Returns `None` if the lengths do not match or scratch memory is unavailable. +pub fn decrypt( + key: &[u8; 32], + nonce: &[u8; 12], + aad: &[u8], + sealed: &[u8], + expected_crc: Option, + out: &mut [u8], +) -> Option { + if sealed.len().checked_sub(16) != Some(out.len()) { + return None; + } + let mut verdict = RawVerdict::default(); + let rc = unsafe { + racer_gcm_host_decrypt( + key, + nonce, + aad.as_ptr(), + aad.len() as u64, + sealed.as_ptr(), + out.len() as u64, + expected_crc.is_some() as u32, + expected_crc.unwrap_or(0), + out.as_mut_ptr(), + &mut verdict, + ) + }; + (rc == 0).then_some(verdict) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + /// FIPS-197 appendix C.3 AES-256 known answer. + fn aes256_known_answer() { + let key: [u8; 32] = std::array::from_fn(|i| i as u8); + let block: [u8; 16] = std::array::from_fn(|i| (i as u8) * 0x11); + assert_eq!( + aes256(&key, &block), + [ + 0x8e, 0xa2, 0xb7, 0xca, 0x51, 0x67, 0x45, 0xbf, 0xea, 0xfc, 0x49, 0x90, 0x4b, 0x49, + 0x60, 0x89 + ] + ); + } + + #[test] + /// The CRC-64/XZ check value, and empty input. + fn crc64_check_value() { + assert_eq!(crc64(b"123456789"), 0x995d_c9bb_df19_39fa); + assert_eq!(crc64(b""), 0); + } + + #[test] + /// NIST GCM test case 14: AES-256, zero key, one zero block, no AAD. + fn gcm_nist_test_case_14() { + let ciphertext = [ + 0xce, 0xa7, 0x40, 0x3d, 0x4d, 0x60, 0x6b, 0x6e, 0x07, 0x4e, 0xc5, 0xd3, 0xba, 0xf3, + 0x9d, 0x18, + ]; + let tag = [ + 0xd0, 0xd1, 0xc8, 0xa7, 0x99, 0x99, 0x6b, 0xf0, 0x26, 0x5b, 0x98, 0xb5, 0xd4, 0x8a, + 0xb9, 0x19, + ]; + let sealed = [ciphertext, tag].concat(); + let mut out = [1; 16]; + let verdict = decrypt(&[0; 32], &[0; 12], &[], &sealed, None, &mut out).unwrap(); + assert_eq!(verdict.status, VERDICT_PASS); + assert_eq!(out, [0; 16]); + } + + #[test] + /// Mismatched output length is refused before any work. + fn decrypt_rejects_length_mismatch() { + let mut out = [0; 3]; + assert!(decrypt(&[0; 32], &[0; 12], &[], &[0; 20], None, &mut out).is_none()); + assert!(decrypt(&[0; 32], &[0; 12], &[], &[0; 2], None, &mut []).is_none()); + } +} diff --git a/cmd/racer-dataplane/gpu/src/lib.rs b/cmd/racer-dataplane/gpu/src/lib.rs new file mode 100644 index 000000000..73a682323 --- /dev/null +++ b/cmd/racer-dataplane/gpu/src/lib.rs @@ -0,0 +1,2 @@ +//! GPU support for Racer (placeholder while the crate is assembled). +pub mod gcm; diff --git a/cmd/racer-dataplane/gpu/tests/gcm_core.rs b/cmd/racer-dataplane/gpu/tests/gcm_core.rs new file mode 100644 index 000000000..0d97bfa13 --- /dev/null +++ b/cmd/racer-dataplane/gpu/tests/gcm_core.rs @@ -0,0 +1,186 @@ +//! Checks the C decrypt core (`native/gcm_core.h`, run serially through the +//! same chunked GHASH and CRC decomposition the CUDA kernels use) against the +//! `racer-crypto` XAES-256-GCM implementation and CRC-64/XZ. + +use racer_gpu::gcm::{self, VERDICT_CRC, VERDICT_PASS, VERDICT_TAG}; + +/// Deterministic xorshift bytes. +struct Rng(u64); + +impl Rng { + /// Fill `out` with pseudo-random bytes. + fn fill(&mut self, out: &mut [u8]) { + for chunk in out.chunks_mut(8) { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + chunk.copy_from_slice(&self.0.to_le_bytes()[..chunk.len()]); + } + } + + /// A random array. + fn array(&mut self) -> [u8; N] { + let mut out = [0; N]; + self.fill(&mut out); + out + } +} + +/// A page sealed the way the dataplane seals pages. +struct Page { + key: [u8; 32], + nonce: [u8; 24], + aad: Vec, + plaintext: Vec, + sealed: Vec, +} + +/// Seal `length` random bytes with XAES-256-GCM and a page-shaped AAD. +fn page(rng: &mut Rng, length: usize) -> Page { + let key = rng.array(); + let nonce = rng.array(); + // Same shape as security::page_aad: domain, cipher, length-prefixed + // fields, page number, key ID, nonce and both lengths. + let mut aad = b"racer/page/aead/v1\0".to_vec(); + aad.extend_from_slice(&1u32.to_be_bytes()); + aad.extend_from_slice(&36u32.to_be_bytes()); + aad.extend_from_slice(b"33333333-3333-4333-8333-333333333333"); + aad.extend_from_slice(&rng.array::<32>()); + aad.extend_from_slice(&4u32.to_be_bytes()); + aad.extend_from_slice(b"\"v1\""); + aad.extend_from_slice(&7u64.to_be_bytes()); + aad.extend_from_slice(&rng.array::<16>()); + aad.extend_from_slice(&nonce); + aad.extend_from_slice(&(length as u32).to_be_bytes()); + aad.extend_from_slice(&(length as u32 + 16).to_be_bytes()); + let mut plaintext = vec![0; length]; + rng.fill(&mut plaintext); + let mut sealed = vec![0; length + racer_crypto::TAG_LEN]; + racer_crypto::seal(&key, &nonce, &aad, &plaintext, &mut sealed).unwrap(); + Page { + key, + nonce, + aad, + plaintext, + sealed, + } +} + +/// Decrypt through the C core with the Rust-derived XAES key. +fn open(page: &Page, aad: &[u8], sealed: &[u8], crc: Option) -> (u32, u64, Vec) { + let derived = racer_crypto::xaes_derive(&page.key, &page.nonce); + let mut out = vec![0xa5; sealed.len() - 16]; + let verdict = gcm::decrypt(derived.key(), derived.nonce(), aad, sealed, crc, &mut out).unwrap(); + (verdict.status, verdict.crc, out) +} + +#[test] +/// Lengths around block and chunk edges, including empty and 16 MiB pages. +fn core_matches_crypto_crate_for_many_lengths() { + let mut rng = Rng(0x1234_5678_9abc_def1); + let chunk = gcm::CHUNK as usize; + let mut lengths = vec![ + 0, + 1, + 15, + 16, + 17, + 31, + 33, + 255, + 4079, + 4080, + 4081, + chunk - 1, + chunk, + chunk + 1, + 3 * chunk - 17, + 65536 + 5, + 1 << 20, + (1 << 20) + 13, + 16 << 20, + ]; + for _ in 0..8 { + let mut b = [0u8; 4]; + rng.fill(&mut b); + lengths.push(u32::from_le_bytes(b) as usize % 200_000); + } + for length in lengths { + let page = page(&mut rng, length); + let expected_crc = racer_crypto::crc64(&page.sealed); + assert_eq!(gcm::crc64(&page.sealed), expected_crc, "length {length}"); + let (status, crc, out) = open(&page, &page.aad, &page.sealed, Some(expected_crc)); + assert_eq!(status, VERDICT_PASS, "length {length}"); + assert_eq!(crc, expected_crc, "length {length}"); + assert!(out == page.plaintext, "length {length}"); + let (status, crc, _) = open(&page, &page.aad, &page.sealed, None); + assert_eq!( + (status, crc), + (VERDICT_PASS, expected_crc), + "length {length}" + ); + } +} + +#[test] +/// Any tamper fails verification and leaves the output untouched. +fn core_rejects_tampering_without_writing() { + let mut rng = Rng(42); + for length in [1, 16, 4097, 70_001] { + let page = page(&mut rng, length); + let crc = racer_crypto::crc64(&page.sealed); + let untouched = vec![0xa5; length]; + + // Flipped ciphertext bit, with and without a matching stored CRC. + let mut sealed = page.sealed.clone(); + sealed[length / 2] ^= 0x10; + let (status, actual, out) = open(&page, &page.aad, &sealed, Some(crc)); + assert_eq!(status, VERDICT_CRC); + assert_eq!(actual, racer_crypto::crc64(&sealed)); + assert_eq!(out, untouched); + let fresh = racer_crypto::crc64(&sealed); + let (status, _, out) = open(&page, &page.aad, &sealed, Some(fresh)); + assert_eq!(status, VERDICT_TAG); + assert_eq!(out, untouched); + + // Flipped tag bit. + let mut sealed = page.sealed.clone(); + sealed[length + 15] ^= 1; + let (status, _, out) = open(&page, &page.aad, &sealed, None); + assert_eq!(status, VERDICT_TAG); + assert_eq!(out, untouched); + + // Wrong AAD. + let mut aad = page.aad.clone(); + aad[20] ^= 1; + let (status, _, out) = open(&page, &aad, &page.sealed, Some(crc)); + assert_eq!(status, VERDICT_TAG); + assert_eq!(out, untouched); + + // Truncated AAD. + let (status, _, _) = open(&page, &page.aad[..page.aad.len() - 1], &page.sealed, None); + assert_eq!(status, VERDICT_TAG); + + // Wrong expected CRC with intact data. + let (status, actual, out) = open(&page, &page.aad, &page.sealed, Some(crc ^ 1)); + assert_eq!((status, actual), (VERDICT_CRC, crc)); + assert_eq!(out, untouched); + + // Wrong key. + let mut other = page.key; + other[0] ^= 1; + let derived = racer_crypto::xaes_derive(&other, &page.nonce); + let mut out = untouched.clone(); + let verdict = gcm::decrypt( + derived.key(), + derived.nonce(), + &page.aad, + &page.sealed, + None, + &mut out, + ) + .unwrap(); + assert_eq!(verdict.status, VERDICT_TAG); + assert_eq!(out, untouched); + } +} From e8f4cbee1aee781b51554f1610a1605c453eb38f Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 19:16:56 +0000 Subject: [PATCH 14/87] feat(racer): add CUDA driver shim and GPU decrypt kernels --- cmd/racer-dataplane/gpu/native/cuda_private.h | 189 +++++ cmd/racer-dataplane/gpu/native/gcm_core.h | 13 +- cmd/racer-dataplane/gpu/native/gcm_kernel.cu | 81 ++ cmd/racer-dataplane/gpu/native/gpu_cuda.c | 757 ++++++++++++++++++ cmd/racer-dataplane/gpu/native/gpu_cuda.h | 154 ++++ images/racer-dataplane/build-native-cuda.sh | 43 + 6 files changed, 1232 insertions(+), 5 deletions(-) create mode 100644 cmd/racer-dataplane/gpu/native/cuda_private.h create mode 100644 cmd/racer-dataplane/gpu/native/gcm_kernel.cu create mode 100644 cmd/racer-dataplane/gpu/native/gpu_cuda.c create mode 100644 cmd/racer-dataplane/gpu/native/gpu_cuda.h create mode 100755 images/racer-dataplane/build-native-cuda.sh diff --git a/cmd/racer-dataplane/gpu/native/cuda_private.h b/cmd/racer-dataplane/gpu/native/cuda_private.h new file mode 100644 index 000000000..bdf9a89b6 --- /dev/null +++ b/cmd/racer-dataplane/gpu/native/cuda_private.h @@ -0,0 +1,189 @@ +/* The few CUDA driver, NVML and cuFile declarations gpu_cuda.c needs. + * + * The shim loads every library with dlopen, so it builds without the CUDA + * toolkit. Values and layouts are copied from the vendor headers: + * + * cuda.h (CUDA 12.9, CUDA_VERSION 12090): + * https://gitlab.com/nvidia/headers/cuda-individual/cudart/-/raw/main/cuda.h + * https://docs.nvidia.com/cuda/cuda-driver-api/ + * nvml.h: + * https://gitlab.com/nvidia/headers/cuda-individual/nvml_dev/-/raw/main/nvml.h + * https://docs.nvidia.com/deploy/nvml-api/ + * cufile.h (GDS cuFile API reference, section 4.1): + * https://docs.nvidia.com/gpudirect-storage/api-reference-guide/index.html + * + * Symbol names include the version suffix cuda.h maps the plain name to (for + * example cuMemAlloc -> cuMemAlloc_v2), because dlsym sees the real export. + * Only append to this file; never change a value without the vendor header. */ +#ifndef RACER_GPU_CUDA_PRIVATE_H +#define RACER_GPU_CUDA_PRIVATE_H + +#include +#include +#include + +/* ---- CUDA driver API (cuda.h) ---- */ +typedef int CUresult; /* enum cudaError_enum; CUDA_SUCCESS = 0 */ +typedef int CUdevice; /* CUdevice_v1 */ +typedef unsigned long long CUdeviceptr; /* CUdeviceptr_v2 on 64-bit hosts */ +typedef unsigned long long CUmemGenericAllocationHandle; +typedef struct CUctx_st *CUcontext; +typedef struct CUstream_st *CUstream; +typedef struct CUmod_st *CUmodule; +typedef struct CUfunc_st *CUfunction; +typedef struct { char bytes[16]; } CUuuid; + +#define CUDA_SUCCESS 0 + +/* CUdevice_attribute */ +#define CU_DEVICE_ATTRIBUTE_COMPUTE_MODE 20 +#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED 102 +#define CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR_SUPPORTED 103 +#define CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_SUPPORTED 116 +#define CU_DEVICE_ATTRIBUTE_DMA_BUF_SUPPORTED 124 + +/* CUstream_flags */ +#define CU_STREAM_NON_BLOCKING 0x1 +/* CUmemAllocationHandleType */ +#define CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR 0x1 +/* CUmemAccess_flags */ +#define CU_MEM_ACCESS_FLAGS_PROT_READWRITE 0x3 +/* CUmemLocationType */ +#define CU_MEM_LOCATION_TYPE_DEVICE 0x1 +/* CUmemAllocationType */ +#define CU_MEM_ALLOCATION_TYPE_PINNED 0x1 +/* CUmemAllocationGranularity_flags */ +#define CU_MEM_ALLOC_GRANULARITY_MINIMUM 0x0 +/* CUmemRangeHandleType */ +#define CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD 0x1 + +typedef struct { + int type; /* CUmemLocationType */ + int id; +} CUmemLocation; + +typedef struct { + int type; /* CUmemAllocationType */ + int requestedHandleTypes; /* CUmemAllocationHandleType */ + CUmemLocation location; + void *win32HandleMetaData; + struct { + unsigned char compressionType; + unsigned char gpuDirectRDMACapable; + unsigned short usage; + unsigned char reserved[4]; + } allocFlags; +} CUmemAllocationProp; + +typedef struct { + CUmemLocation location; + int flags; /* CUmemAccess_flags */ +} CUmemAccessDesc; + +_Static_assert(sizeof(CUmemLocation) == 8, "CUmemLocation layout"); +_Static_assert(sizeof(CUmemAllocationProp) == 32, "CUmemAllocationProp layout"); +_Static_assert(offsetof(CUmemAllocationProp, win32HandleMetaData) == 16, "CUmemAllocationProp layout"); +_Static_assert(sizeof(CUmemAccessDesc) == 12, "CUmemAccessDesc layout"); + +/* X(name, symbol, return, arguments) for every driver function. */ +#define RACER_CUDA_FUNCTIONS(X) \ + X(cuInit, "cuInit", CUresult, (unsigned int flags)) \ + X(cuDeviceGetCount, "cuDeviceGetCount", CUresult, (int *count)) \ + X(cuDeviceGet, "cuDeviceGet", CUresult, (CUdevice *device, int ordinal)) \ + X(cuDeviceGetUuid, "cuDeviceGetUuid_v2", CUresult, (CUuuid *uuid, CUdevice dev)) \ + X(cuDeviceGetPCIBusId, "cuDeviceGetPCIBusId", CUresult, (char *id, int len, CUdevice dev)) \ + X(cuDeviceGetAttribute, "cuDeviceGetAttribute", CUresult, (int *pi, int attrib, CUdevice dev)) \ + X(cuDeviceTotalMem, "cuDeviceTotalMem_v2", CUresult, (size_t *bytes, CUdevice dev)) \ + X(cuDevicePrimaryCtxRetain, "cuDevicePrimaryCtxRetain", CUresult, (CUcontext *ctx, CUdevice dev)) \ + X(cuDevicePrimaryCtxRelease, "cuDevicePrimaryCtxRelease_v2", CUresult, (CUdevice dev)) \ + X(cuCtxPushCurrent, "cuCtxPushCurrent_v2", CUresult, (CUcontext ctx)) \ + X(cuCtxPopCurrent, "cuCtxPopCurrent_v2", CUresult, (CUcontext *ctx)) \ + X(cuMemImportFromShareableHandle, "cuMemImportFromShareableHandle", CUresult, \ + (CUmemGenericAllocationHandle *handle, void *os_handle, int type)) \ + X(cuMemGetAllocationPropertiesFromHandle, "cuMemGetAllocationPropertiesFromHandle", CUresult, \ + (CUmemAllocationProp *prop, CUmemGenericAllocationHandle handle)) \ + X(cuMemGetAllocationGranularity, "cuMemGetAllocationGranularity", CUresult, \ + (size_t *granularity, const CUmemAllocationProp *prop, int option)) \ + X(cuMemAddressReserve, "cuMemAddressReserve", CUresult, \ + (CUdeviceptr *ptr, size_t size, size_t alignment, CUdeviceptr addr, unsigned long long flags)) \ + X(cuMemAddressFree, "cuMemAddressFree", CUresult, (CUdeviceptr ptr, size_t size)) \ + X(cuMemMap, "cuMemMap", CUresult, \ + (CUdeviceptr ptr, size_t size, size_t offset, CUmemGenericAllocationHandle handle, \ + unsigned long long flags)) \ + X(cuMemUnmap, "cuMemUnmap", CUresult, (CUdeviceptr ptr, size_t size)) \ + X(cuMemSetAccess, "cuMemSetAccess", CUresult, \ + (CUdeviceptr ptr, size_t size, const CUmemAccessDesc *desc, size_t count)) \ + X(cuMemRelease, "cuMemRelease", CUresult, (CUmemGenericAllocationHandle handle)) \ + X(cuMemAlloc, "cuMemAlloc_v2", CUresult, (CUdeviceptr *ptr, size_t size)) \ + X(cuMemFree, "cuMemFree_v2", CUresult, (CUdeviceptr ptr)) \ + X(cuMemAllocHost, "cuMemAllocHost_v2", CUresult, (void **ptr, size_t size)) \ + X(cuMemFreeHost, "cuMemFreeHost", CUresult, (void *ptr)) \ + X(cuMemcpyHtoDAsync, "cuMemcpyHtoDAsync_v2", CUresult, \ + (CUdeviceptr dst, const void *src, size_t n, CUstream stream)) \ + X(cuMemcpyDtoDAsync, "cuMemcpyDtoDAsync_v2", CUresult, \ + (CUdeviceptr dst, CUdeviceptr src, size_t n, CUstream stream)) \ + X(cuMemcpyDtoHAsync, "cuMemcpyDtoHAsync_v2", CUresult, \ + (void *dst, CUdeviceptr src, size_t n, CUstream stream)) \ + X(cuMemsetD8Async, "cuMemsetD8Async", CUresult, \ + (CUdeviceptr dst, unsigned char value, size_t n, CUstream stream)) \ + X(cuStreamCreate, "cuStreamCreate", CUresult, (CUstream *stream, unsigned int flags)) \ + X(cuStreamSynchronize, "cuStreamSynchronize", CUresult, (CUstream stream)) \ + X(cuStreamDestroy, "cuStreamDestroy_v2", CUresult, (CUstream stream)) \ + X(cuModuleLoadData, "cuModuleLoadData", CUresult, (CUmodule *module, const void *image)) \ + X(cuModuleUnload, "cuModuleUnload", CUresult, (CUmodule module)) \ + X(cuModuleGetFunction, "cuModuleGetFunction", CUresult, \ + (CUfunction *fn, CUmodule module, const char *name)) \ + X(cuLaunchKernel, "cuLaunchKernel", CUresult, \ + (CUfunction f, unsigned int gx, unsigned int gy, unsigned int gz, unsigned int bx, \ + unsigned int by, unsigned int bz, unsigned int shared, CUstream stream, void **params, \ + void **extra)) \ + X(cuMemGetHandleForAddressRange, "cuMemGetHandleForAddressRange", CUresult, \ + (void *handle, CUdeviceptr ptr, size_t size, int type, unsigned long long flags)) + +/* ---- NVML (nvml.h), optional; used only for the NUMA node ---- */ +typedef int nvmlReturn_t; /* NVML_SUCCESS = 0 */ +typedef struct nvmlDevice_st *nvmlDevice_t; +#define NVML_SUCCESS 0 + +#define RACER_NVML_FUNCTIONS(X) \ + X(nvmlInit, "nvmlInit_v2", nvmlReturn_t, (void)) \ + X(nvmlShutdown, "nvmlShutdown", nvmlReturn_t, (void)) \ + X(nvmlDeviceGetHandleByPciBusId, "nvmlDeviceGetHandleByPciBusId_v2", nvmlReturn_t, \ + (const char *id, nvmlDevice_t *device)) \ + X(nvmlDeviceGetNumaNodeId, "nvmlDeviceGetNumaNodeId", nvmlReturn_t, \ + (nvmlDevice_t device, unsigned int *node)) + +/* ---- cuFile (cufile.h), optional ---- */ +typedef struct { + int err; /* CUfileOpError; CU_FILE_SUCCESS = 0 */ + int cu_err; /* CUresult */ +} CUfileError_t; +typedef void *CUfileHandle_t; +typedef struct { + int type; /* CUfileFileHandleType */ + union { + int fd; + void *handle; + } handle; + const void *fs_ops; +} CUfileDescr_t; + +#define CU_FILE_SUCCESS 0 +#define CU_FILE_HANDLE_TYPE_OPAQUE_FD 1 + +_Static_assert(sizeof(CUfileError_t) == 8, "CUfileError_t layout"); +_Static_assert(sizeof(CUfileDescr_t) == 24, "CUfileDescr_t layout"); +_Static_assert(offsetof(CUfileDescr_t, handle) == 8, "CUfileDescr_t layout"); + +#define RACER_CUFILE_FUNCTIONS(X) \ + X(cuFileDriverOpen, "cuFileDriverOpen", CUfileError_t, (void)) \ + X(cuFileHandleRegister, "cuFileHandleRegister", CUfileError_t, \ + (CUfileHandle_t *fh, CUfileDescr_t *descr)) \ + X(cuFileHandleDeregister, "cuFileHandleDeregister", void, (CUfileHandle_t fh)) \ + X(cuFileBufRegister, "cuFileBufRegister", CUfileError_t, \ + (const void *base, size_t size, int flags)) \ + X(cuFileBufDeregister, "cuFileBufDeregister", CUfileError_t, (const void *base)) \ + X(cuFileRead, "cuFileRead", ssize_t, \ + (CUfileHandle_t fh, void *base, size_t size, off_t file_offset, off_t base_offset)) + +#endif diff --git a/cmd/racer-dataplane/gpu/native/gcm_core.h b/cmd/racer-dataplane/gpu/native/gcm_core.h index a4ca5ee98..18451be32 100644 --- a/cmd/racer-dataplane/gpu/native/gcm_core.h +++ b/cmd/racer-dataplane/gpu/native/gcm_core.h @@ -209,13 +209,16 @@ GCM_FN void gcm_aes256_encrypt(const uint8_t *sbox, const uint8_t *rk, const uin for (int i = 0; i < 16; ++i) out[i] = s[i]; } +/* Entry `i` of the reflected CRC-64/XZ byte table. */ +GCM_FN uint64_t gcm_crc64_entry(unsigned i) { + uint64_t c = i; + for (int k = 0; k < 8; ++k) c = (c & 1) ? (c >> 1) ^ GCM_CRC64_POLY : c >> 1; + return c; +} + /* Reflected CRC-64/XZ byte table. */ GCM_FN void gcm_crc64_table(uint64_t table[256]) { - for (unsigned i = 0; i < 256; ++i) { - uint64_t c = i; - for (int k = 0; k < 8; ++k) c = (c & 1) ? (c >> 1) ^ GCM_CRC64_POLY : c >> 1; - table[i] = c; - } + for (unsigned i = 0; i < 256; ++i) table[i] = gcm_crc64_entry(i); } /* Complete CRC-64/XZ (init and xorout all ones) of `n` bytes. */ diff --git a/cmd/racer-dataplane/gpu/native/gcm_kernel.cu b/cmd/racer-dataplane/gpu/native/gcm_kernel.cu new file mode 100644 index 000000000..35cb78fcc --- /dev/null +++ b/cmd/racer-dataplane/gpu/native/gcm_kernel.cu @@ -0,0 +1,81 @@ +/* CUDA kernels for GPU page decryption. See gcm_core.h for the math. + * + * Built to PTX by images/racer-dataplane/build-native-cuda.sh with + * `nvcc -ptx -arch=compute_80`. compute_80 (Ampere) is the baseline: the + * driver JIT-compiles the PTX for the installed GPU, so newer GPUs work and + * older ones fail at cuModuleLoadData (the shim then reports no kernels). + * + * Launch order on one stream, all set up by racer_gpu_decrypt in gpu_cuda.c: + * 1. racer_gcm_partials one thread per GCM_CHUNK of the sealed page + * 2. racer_gcm_finalize one block of 256 threads; writes the verdict + * 3. racer_gcm_ctr one thread per 16-byte block; writes plaintext only + * when the verdict on the device is GCM_VERDICT_PASS + * No host round trip sits between verification and decryption. */ +#include "gcm_core.h" + +__device__ const uint8_t racer_gcm_sbox[256] = GCM_SBOX_INIT; + +#define FINALIZE_THREADS 256 + +extern "C" __global__ void racer_gcm_partials(const uint8_t *sealed, uint64_t n, + const struct gcm_key *key, + struct gcm_partial *partials, uint64_t chunks) { + __shared__ uint64_t table[256]; + for (unsigned i = threadIdx.x; i < 256; i += blockDim.x) table[i] = gcm_crc64_entry(i); + __syncthreads(); + uint64_t j = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (j < chunks) gcm_chunk_partial(table, key, sealed, n, j, &partials[j]); +} + +extern "C" __global__ void racer_gcm_finalize(const uint8_t *sealed, uint64_t n, + const struct gcm_key *key, + const struct gcm_partial *partials, uint64_t chunks, + uint32_t check_crc, uint64_t expected_crc, + struct gcm_verdict *verdict) { + __shared__ uint64_t hi[FINALIZE_THREADS], lo[FINALIZE_THREADS], crc[FINALIZE_THREADS]; + unsigned t = threadIdx.x; + uint64_t h = 0, l = 0, c = 0; + for (uint64_t j = t; j < chunks; j += FINALIZE_THREADS) { + struct gcm_block b = gcm_load(partials[j].ghash); + h ^= b.hi; + l ^= b.lo; + c ^= partials[j].crc; + } + hi[t] = h; + lo[t] = l; + crc[t] = c; + __syncthreads(); + for (unsigned stride = FINALIZE_THREADS / 2; stride; stride >>= 1) { + if (t < stride) { + hi[t] ^= hi[t + stride]; + lo[t] ^= lo[t + stride]; + crc[t] ^= crc[t + stride]; + } + __syncthreads(); + } + if (t == 0) { + uint8_t ghash[16]; + struct gcm_block b = {hi[0], lo[0]}; + gcm_store(ghash, b); + gcm_finalize(key, sealed, n, ghash, crc[0], check_crc, expected_crc, verdict); + } +} + +extern "C" __global__ void racer_gcm_ctr(const uint8_t *sealed, uint64_t n, + const struct gcm_key *key, + const struct gcm_verdict *verdict, uint8_t *out) { + __shared__ uint8_t sbox[256]; + __shared__ struct gcm_key k; + /* Same value for every thread, so the whole block leaves together. */ + if (verdict->status != GCM_VERDICT_PASS) return; + uint8_t *raw = (uint8_t *)&k; + const uint8_t *from = (const uint8_t *)key; + for (unsigned i = threadIdx.x; i < 256; i += blockDim.x) sbox[i] = racer_gcm_sbox[i]; + for (unsigned i = threadIdx.x; i < sizeof(k); i += blockDim.x) raw[i] = from[i]; + __syncthreads(); + uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < (n + 15) / 16) gcm_ctr_block(sbox, &k, sealed, n, i, out); + __syncthreads(); + /* Shared memory outlives the block; do not leave round keys behind. */ + for (unsigned j = threadIdx.x; j < sizeof(k); j += blockDim.x) raw[j] = 0; +} diff --git a/cmd/racer-dataplane/gpu/native/gpu_cuda.c b/cmd/racer-dataplane/gpu/native/gpu_cuda.c new file mode 100644 index 000000000..b2a75a286 --- /dev/null +++ b/cmd/racer-dataplane/gpu/native/gpu_cuda.c @@ -0,0 +1,757 @@ +/* Thin C shim between racer-gpu (Rust) and the CUDA driver API, NVML and + * cuFile. Rust loads it as libracer_gpu_cuda.so.1; see gpu_cuda.h for the + * ABI and result contract. + * + * The shim itself dlopens libcuda.so.1, libnvidia-ml.so.1 and libcufile.so.0 + * with the declarations in cuda_private.h, so it builds without the CUDA + * toolkit. The decrypt kernels are PTX embedded at build time when + * RACER_GPU_PTX names a PTX file (see build-native-cuda.sh); without it, every + * other call works and racer_gpu_decrypt returns RACER_GPU_ENOKERNEL. + * + * Every call runs on the dataplane's native helper thread. Each call pushes + * the owning context and pops it before returning. */ +#define _GNU_SOURCE +#include "gpu_cuda.h" +#include "cuda_private.h" +#include "gcm_core.h" + +#include +#include +#include +#include +#include +#include + +#ifdef RACER_GPU_PTX +/* Embed the PTX image, NUL terminated as cuModuleLoadData requires. */ +__asm__(".section .rodata.racer_gpu_ptx,\"a\"\n" + ".globl racer_gpu_ptx_image\n" + ".hidden racer_gpu_ptx_image\n" + "racer_gpu_ptx_image:\n" + ".incbin \"" RACER_GPU_PTX "\"\n" + ".byte 0\n" + ".previous\n"); +extern const char racer_gpu_ptx_image[]; +#define PTX_IMAGE racer_gpu_ptx_image +#else +#define PTX_IMAGE NULL +#endif + +/* Function pointer types and tables for each library. */ +#define X_TYPE(name, symbol, ret, args) typedef ret(*name##_fn) args; +#define X_FIELD(name, symbol, ret, args) name##_fn name; +#define X_LOAD(name, symbol, ret, args) \ + if (!(table.name = (name##_fn)dlsym(table.lib, symbol))) goto missing; +RACER_CUDA_FUNCTIONS(X_TYPE) +RACER_NVML_FUNCTIONS(X_TYPE) +RACER_CUFILE_FUNCTIONS(X_TYPE) + +struct cuda_table { void *lib; RACER_CUDA_FUNCTIONS(X_FIELD) }; +struct nvml_table { void *lib; RACER_NVML_FUNCTIONS(X_FIELD) }; +struct cufile_table { void *lib; int opened; RACER_CUFILE_FUNCTIONS(X_FIELD) }; + +static struct cuda_table cu; +static struct cufile_table cf; +static pthread_mutex_t load_lock = PTHREAD_MUTEX_INITIALIZER; +static __thread int32_t last_error; + +static const uint8_t shim_sbox[256] = GCM_SBOX_INIT; + +#define KIND_IMPORT 0x494d5054u +#define KIND_STAGING 0x53544147u +#define STAGING_ALIGN (2ull << 20) +/* Workspace layout: key, verdict, then one partial per chunk. */ +#define WORK_KEY 0u +#define WORK_VERDICT 512u +#define WORK_PARTIALS 1024u +#define PARTIAL_THREADS 128u +#define FINALIZE_THREADS 256u +#define CTR_THREADS 256u + +_Static_assert(sizeof(struct gcm_key) <= WORK_VERDICT, "workspace key slot"); +_Static_assert(sizeof(struct gcm_partial) == 32, "partial layout"); +_Static_assert(sizeof(struct gcm_verdict) == sizeof(struct racer_gpu_verdict), "verdict layout"); + +struct gpu_context { + CUcontext ctx; + CUdevice dev; + int32_t ordinal; + int module_state; /* 0 untried, 1 loaded, -1 unavailable */ + CUmodule module; + CUfunction partials, finalize, ctr; +}; + +struct gpu_buffer { + uint32_t kind; + struct gpu_context *context; + CUdeviceptr base; + uint64_t size; +}; + +struct gpu_import { + struct gpu_buffer b; + CUmemGenericAllocationHandle handle; + int has_handle, reserved, mapped; +}; + +struct gpu_staging { + struct gpu_buffer b; + uint64_t allocated; + int registered; +}; + +struct gpu_stream { + struct gpu_context *context; + CUstream stream; + CUdeviceptr work; + uint64_t work_size; + struct racer_gpu_verdict *verdict; /* pinned host memory */ + int decrypt; /* 0 none, 1 queued, 2 synchronized */ +}; + +struct gpu_file { + CUfileHandle_t handle; +}; + +uint32_t racer_gpu_abi(void) { return RACER_GPU_ABI_VERSION; } + +int32_t racer_gpu_last_driver_error(void) { return last_error; } + +/* Map a driver result to a shim code, remembering the raw value. */ +static int drv(CUresult result) { + if (result == CUDA_SUCCESS) return RACER_GPU_OK; + last_error = result; + return RACER_GPU_EDRIVER; +} + +static void wipe(void *p, size_t n) { + volatile uint8_t *v = (volatile uint8_t *)p; + while (n--) *v++ = 0; +} + +int racer_gpu_init(void) { + int rc = RACER_GPU_OK; + pthread_mutex_lock(&load_lock); + if (!cu.lib) { + struct cuda_table table; + memset(&table, 0, sizeof(table)); + table.lib = dlopen("libcuda.so.1", RTLD_NOW | RTLD_LOCAL); + if (!table.lib) { + rc = RACER_GPU_EUNAVAILABLE; + goto out; + } + RACER_CUDA_FUNCTIONS(X_LOAD) + if (drv(table.cuInit(0))) { + dlclose(table.lib); + rc = RACER_GPU_EUNAVAILABLE; + goto out; + } + cu = table; + goto out; + missing: + dlclose(table.lib); + rc = RACER_GPU_EUNAVAILABLE; + } +out: + pthread_mutex_unlock(&load_lock); + return rc; +} + +/* Load NVML for one discovery. Returns 0 when usable. */ +static int nvml_open(struct nvml_table *out) { + struct nvml_table table; + memset(&table, 0, sizeof(table)); + table.lib = dlopen("libnvidia-ml.so.1", RTLD_NOW | RTLD_LOCAL); + if (!table.lib) return -1; + RACER_NVML_FUNCTIONS(X_LOAD) + if (table.nvmlInit() != NVML_SUCCESS) goto missing; + *out = table; + return 0; +missing: + dlclose(table.lib); + return -1; +} + +/* NUMA node from sysfs for a lowercase PCI address, or -1. */ +static int32_t sysfs_numa(const char *pci) { + char path[96]; + int node = -1; + if (snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/numa_node", pci) >= (int)sizeof(path)) + return -1; + FILE *f = fopen(path, "re"); + if (!f) return -1; + if (fscanf(f, "%d", &node) != 1) node = -1; + fclose(f); + return node < 0 ? -1 : node; +} + +static int describe(int ordinal, struct racer_gpu_device *d, struct nvml_table *nvml, int have_nvml) { + CUdevice dev; + CUuuid uuid; + size_t total = 0; + int value = 0; + memset(d, 0, sizeof(*d)); + d->ordinal = ordinal; + d->numa_node = -1; + d->compute_mode = -1; + if (drv(cu.cuDeviceGet(&dev, ordinal)) || drv(cu.cuDeviceGetUuid(&uuid, dev)) || + drv(cu.cuDeviceGetPCIBusId(d->pci_bus_id, sizeof(d->pci_bus_id), dev)) || + drv(cu.cuDeviceTotalMem(&total, dev))) + return -1; + d->pci_bus_id[sizeof(d->pci_bus_id) - 1] = 0; + for (char *p = d->pci_bus_id; *p; ++p) + if (*p >= 'A' && *p <= 'F') *p = (char)(*p - 'A' + 'a'); + memcpy(d->uuid, uuid.bytes, 16); + d->total_memory = total; + static const struct { int attribute; uint32_t flag; } caps[] = { + {CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, RACER_GPU_FLAG_VMM}, + {CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR_SUPPORTED, RACER_GPU_FLAG_POSIX_FD}, + {CU_DEVICE_ATTRIBUTE_DMA_BUF_SUPPORTED, RACER_GPU_FLAG_DMA_BUF}, + {CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_SUPPORTED, RACER_GPU_FLAG_GDR}, + }; + for (size_t i = 0; i < sizeof(caps) / sizeof(caps[0]); ++i) + if (!cu.cuDeviceGetAttribute(&value, caps[i].attribute, dev) && value) d->flags |= caps[i].flag; + unsigned int node; + nvmlDevice_t handle; + if (have_nvml && nvml->nvmlDeviceGetHandleByPciBusId(d->pci_bus_id, &handle) == NVML_SUCCESS && + nvml->nvmlDeviceGetNumaNodeId(handle, &node) == NVML_SUCCESS && node <= INT32_MAX) + d->numa_node = (int32_t)node; + else + d->numa_node = sysfs_numa(d->pci_bus_id); + /* Last, so any earlier failure leaves the device unusable (-1). */ + if (drv(cu.cuDeviceGetAttribute(&value, CU_DEVICE_ATTRIBUTE_COMPUTE_MODE, dev))) return -1; + d->compute_mode = value; + return 0; +} + +int racer_gpu_discover(struct racer_gpu_device *out, uint32_t capacity) { + int count = 0; + struct nvml_table nvml; + if (!cu.lib) return RACER_GPU_EUNAVAILABLE; + if (drv(cu.cuDeviceGetCount(&count))) return RACER_GPU_EDRIVER; + int have_nvml = nvml_open(&nvml) == 0; + for (int i = 0; i < count && (uint32_t)i < capacity; ++i) (void)describe(i, &out[i], &nvml, have_nvml); + if (have_nvml) { + (void)nvml.nvmlShutdown(); + dlclose(nvml.lib); + } + return count; +} + +static int push(struct gpu_context *c) { return drv(cu.cuCtxPushCurrent(c->ctx)); } + +static void pop(void) { + CUcontext previous; + (void)cu.cuCtxPopCurrent(&previous); +} + +int racer_gpu_context(int32_t ordinal, void **out) { + if (!cu.lib) return RACER_GPU_EUNAVAILABLE; + if (!out || ordinal < 0) return RACER_GPU_EINVAL; + *out = NULL; + struct gpu_context *c = calloc(1, sizeof(*c)); + if (!c) return RACER_GPU_ENOMEM; + c->ordinal = ordinal; + int rc = drv(cu.cuDeviceGet(&c->dev, ordinal)); + if (!rc) rc = drv(cu.cuDevicePrimaryCtxRetain(&c->ctx, c->dev)); + if (rc) { + free(c); + return rc; + } + *out = c; + return RACER_GPU_OK; +} + +int racer_gpu_context_free(void *context) { + struct gpu_context *c = context; + if (!c) return RACER_GPU_EINVAL; + if (c->module_state == 1) { + int rc = push(c); + if (rc) return rc; + rc = drv(cu.cuModuleUnload(c->module)); + pop(); + if (rc) return rc; + c->module_state = 0; + } + int rc = drv(cu.cuDevicePrimaryCtxRelease(c->dev)); + if (!rc) free(c); + return rc; +} + +/* Load the decrypt kernels into a context. Context must be current. */ +static int kernels(struct gpu_context *c) { + if (c->module_state == 0) { + const char *image = PTX_IMAGE; + c->module_state = -1; + if (!image) return RACER_GPU_ENOKERNEL; + if (drv(cu.cuModuleLoadData(&c->module, image))) return RACER_GPU_ENOKERNEL; + if (drv(cu.cuModuleGetFunction(&c->partials, c->module, "racer_gcm_partials")) || + drv(cu.cuModuleGetFunction(&c->finalize, c->module, "racer_gcm_finalize")) || + drv(cu.cuModuleGetFunction(&c->ctr, c->module, "racer_gcm_ctr"))) { + /* Keep it as loaded (state 1) if unload fails so free retries. */ + if (drv(cu.cuModuleUnload(c->module))) c->module_state = 1; + return RACER_GPU_ENOKERNEL; + } + c->module_state = 1; + } + return c->module_state == 1 && c->ctr ? RACER_GPU_OK : RACER_GPU_ENOKERNEL; +} + +int racer_gpu_stream(void *context, void **out) { + struct gpu_context *c = context; + if (!c || !out) return RACER_GPU_EINVAL; + *out = NULL; + struct gpu_stream *s = calloc(1, sizeof(*s)); + if (!s) return RACER_GPU_ENOMEM; + s->context = c; + int rc = push(c); + if (rc) { + free(s); + return rc; + } + rc = drv(cu.cuStreamCreate(&s->stream, CU_STREAM_NON_BLOCKING)); + if (!rc) { + rc = drv(cu.cuMemAllocHost((void **)&s->verdict, sizeof(*s->verdict))); + if (rc && drv(cu.cuStreamDestroy(s->stream))) rc = RACER_GPU_ELEAKED; + } + pop(); + if (rc) { + free(s); + return rc; + } + memset(s->verdict, 0, sizeof(*s->verdict)); + *out = s; + return RACER_GPU_OK; +} + +int racer_gpu_stream_sync(void *stream) { + struct gpu_stream *s = stream; + if (!s) return RACER_GPU_EINVAL; + int rc = push(s->context); + if (rc) return rc; + rc = drv(cu.cuStreamSynchronize(s->stream)); + pop(); + if (!rc && s->decrypt == 1) s->decrypt = 2; + return rc; +} + +int racer_gpu_stream_free(void *stream) { + struct gpu_stream *s = stream; + if (!s) return RACER_GPU_EINVAL; + int rc = push(s->context); + if (rc) return rc; + rc = drv(cu.cuStreamSynchronize(s->stream)); + if (!rc && s->work) { + rc = drv(cu.cuMemFree(s->work)); + if (!rc) s->work = 0, s->work_size = 0; + } + if (!rc && s->verdict) { + rc = drv(cu.cuMemFreeHost(s->verdict)); + if (!rc) s->verdict = NULL; + } + if (!rc) rc = drv(cu.cuStreamDestroy(s->stream)); + pop(); + if (!rc) free(s); + return rc; +} + +/* Undo an import as far as possible. Context must be current. */ +static int import_release(struct gpu_import *a) { + if (a->mapped) { + if (drv(cu.cuMemUnmap(a->b.base, a->b.size))) return RACER_GPU_EDRIVER; + a->mapped = 0; + } + if (a->reserved) { + if (drv(cu.cuMemAddressFree(a->b.base, a->b.size))) return RACER_GPU_EDRIVER; + a->reserved = 0; + } + if (a->has_handle) { + if (drv(cu.cuMemRelease(a->handle))) return RACER_GPU_EDRIVER; + a->has_handle = 0; + } + return RACER_GPU_OK; +} + +int racer_gpu_import(void *context, int fd, uint64_t size, void **out) { + struct gpu_context *c = context; + if (!c || fd < 0 || !size || size > SIZE_MAX || !out) return RACER_GPU_EINVAL; + *out = NULL; + struct gpu_import *a = calloc(1, sizeof(*a)); + if (!a) return RACER_GPU_ENOMEM; + a->b.kind = KIND_IMPORT; + a->b.context = c; + a->b.size = size; + int rc = push(c); + if (rc) { + free(a); + return rc; + } + CUmemAllocationProp prop; + size_t granularity = 0; + memset(&prop, 0, sizeof(prop)); + rc = drv(cu.cuMemImportFromShareableHandle(&a->handle, (void *)(uintptr_t)fd, + CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR)); + if (rc) goto fail; + a->has_handle = 1; + rc = drv(cu.cuMemGetAllocationPropertiesFromHandle(&prop, a->handle)); + if (rc) goto fail; + if (prop.location.type != CU_MEM_LOCATION_TYPE_DEVICE || prop.location.id != c->ordinal) { + rc = RACER_GPU_EDEVICE; + goto fail; + } + rc = drv(cu.cuMemGetAllocationGranularity(&granularity, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM)); + if (rc) goto fail; + if (!granularity || size % granularity) { + rc = RACER_GPU_EINVAL; + goto fail; + } + rc = drv(cu.cuMemAddressReserve(&a->b.base, size, 0, 0, 0)); + if (rc) goto fail; + a->reserved = 1; + /* Fails if `size` exceeds the allocation, so we never map past it. */ + rc = drv(cu.cuMemMap(a->b.base, size, 0, a->handle, 0)); + if (rc) goto fail; + a->mapped = 1; + CUmemAccessDesc access; + access.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + access.location.id = c->ordinal; + access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; + rc = drv(cu.cuMemSetAccess(a->b.base, size, &access, 1)); + if (rc) goto fail; + pop(); + *out = a; + return RACER_GPU_OK; +fail: + if (import_release(a)) rc = RACER_GPU_ELEAKED; + else free(a); + pop(); + return rc; +} + +int racer_gpu_import_free(void *allocation) { + struct gpu_import *a = allocation; + if (!a || a->b.kind != KIND_IMPORT) return RACER_GPU_EINVAL; + int rc = push(a->b.context); + if (rc) return rc; + rc = import_release(a); + pop(); + if (!rc) { + a->b.kind = 0; + free(a); + } + return rc; +} + +int racer_gpu_staging(void *context, uint64_t size, void **out) { + struct gpu_context *c = context; + if (!c || !size || size > UINT64_MAX - STAGING_ALIGN || !out) return RACER_GPU_EINVAL; + *out = NULL; + uint64_t allocated = (size + STAGING_ALIGN - 1) & ~(STAGING_ALIGN - 1); + if (allocated > SIZE_MAX) return RACER_GPU_EINVAL; + struct gpu_staging *b = calloc(1, sizeof(*b)); + if (!b) return RACER_GPU_ENOMEM; + b->b.kind = KIND_STAGING; + b->b.context = c; + b->b.size = size; + b->allocated = allocated; + int rc = push(c); + if (rc) { + free(b); + return rc; + } + rc = drv(cu.cuMemAlloc(&b->b.base, allocated)); + /* dma-buf export and cuFile want page alignment; 2 MiB sizes get 2 MiB + * aligned addresses in practice, but check rather than assume. */ + if (!rc && b->b.base % 65536) rc = drv(cu.cuMemFree(b->b.base)) ? RACER_GPU_ELEAKED : RACER_GPU_EINVAL; + pop(); + if (rc) { + if (rc != RACER_GPU_ELEAKED) free(b); + return rc; + } + *out = b; + return RACER_GPU_OK; +} + +int racer_gpu_staging_free(void *staging) { + struct gpu_staging *b = staging; + if (!b || b->b.kind != KIND_STAGING) return RACER_GPU_EINVAL; + if (b->registered) return RACER_GPU_EBUSY; + int rc = push(b->b.context); + if (rc) return rc; + rc = drv(cu.cuMemFree(b->b.base)); + pop(); + if (!rc) { + b->b.kind = 0; + free(b); + } + return rc; +} + +/* Nonempty [offset, offset + length) inside the buffer. */ +static int span(const struct gpu_buffer *b, uint64_t offset, uint64_t length) { + return length && offset <= b->size && length <= b->size - offset ? RACER_GPU_OK : RACER_GPU_ERANGE; +} + +static int buffer(void *p, struct gpu_buffer **out) { + struct gpu_buffer *b = p; + if (!b || (b->kind != KIND_IMPORT && b->kind != KIND_STAGING)) return RACER_GPU_EINVAL; + *out = b; + return RACER_GPU_OK; +} + +int racer_gpu_copy_h2d(void *stream, void *dst, uint64_t offset, const void *src, uint64_t length) { + struct gpu_stream *s = stream; + struct gpu_buffer *d; + if (!s || !src || buffer(dst, &d)) return RACER_GPU_EINVAL; + if (d->context != s->context) return RACER_GPU_EDEVICE; + int rc = span(d, offset, length); + if (rc) return rc; + if ((rc = push(s->context))) return rc; + rc = drv(cu.cuMemcpyHtoDAsync(d->base + offset, src, length, s->stream)); + pop(); + return rc; +} + +int racer_gpu_copy_d2d(void *stream, void *dst, uint64_t dst_offset, void *src, uint64_t src_offset, + uint64_t length) { + struct gpu_stream *s = stream; + struct gpu_buffer *d, *from; + if (!s || buffer(dst, &d) || buffer(src, &from) || from->kind != KIND_STAGING) return RACER_GPU_EINVAL; + if (d->context != s->context || from->context != s->context) return RACER_GPU_EDEVICE; + int rc = span(d, dst_offset, length); + if (!rc) rc = span(from, src_offset, length); + if (rc) return rc; + if ((rc = push(s->context))) return rc; + rc = drv(cu.cuMemcpyDtoDAsync(d->base + dst_offset, from->base + src_offset, length, s->stream)); + pop(); + return rc; +} + +int racer_gpu_copy_d2h(void *stream, void *dst, void *src, uint64_t src_offset, uint64_t length) { + struct gpu_stream *s = stream; + struct gpu_buffer *from; + if (!s || !dst || buffer(src, &from) || from->kind != KIND_STAGING) return RACER_GPU_EINVAL; + if (from->context != s->context) return RACER_GPU_EDEVICE; + int rc = span(from, src_offset, length); + if (rc) return rc; + if ((rc = push(s->context))) return rc; + rc = drv(cu.cuMemcpyDtoHAsync(dst, from->base + src_offset, length, s->stream)); + if (!rc) rc = drv(cu.cuStreamSynchronize(s->stream)); + pop(); + if (!rc && s->decrypt == 1) s->decrypt = 2; + return rc; +} + +static int launch(CUfunction f, uint64_t grid, unsigned threads, CUstream stream, void **args) { + return drv(cu.cuLaunchKernel(f, (unsigned)grid, 1, 1, threads, 1, 1, 0, stream, args, NULL)); +} + +int racer_gpu_decrypt(void *stream, const struct racer_gpu_decrypt *p, void *src, uint64_t src_offset, + void *dst, uint64_t dst_offset) { + struct gpu_stream *s = stream; + struct gpu_buffer *d, *from; + if (!s || !p || buffer(dst, &d) || buffer(src, &from) || from->kind != KIND_STAGING) + return RACER_GPU_EINVAL; + if (p->length > RACER_GPU_MAX_DECRYPT || p->aad_length > RACER_GPU_MAX_AAD || + (!p->aad && p->aad_length) || p->check_crc > 1) + return RACER_GPU_EINVAL; + if (d->context != s->context || from->context != s->context) return RACER_GPU_EDEVICE; + if (s->decrypt == 1) return RACER_GPU_EBUSY; + uint64_t n = p->length; + int rc = span(from, src_offset, n + 16); + if (!rc && n) rc = span(d, dst_offset, n); + if (!rc && !n && dst_offset > d->size) rc = RACER_GPU_ERANGE; + if (rc) return rc; + if ((rc = push(s->context))) return rc; + if ((rc = kernels(s->context))) goto out; + + uint64_t chunks = gcm_chunks(n); + uint64_t need = WORK_PARTIALS + chunks * sizeof(struct gcm_partial); + if (s->work_size < need) { + /* Earlier work may still read the old workspace. */ + if ((rc = drv(cu.cuStreamSynchronize(s->stream)))) goto out; + if (s->work) { + if ((rc = drv(cu.cuMemFree(s->work)))) goto out; + s->work = 0; + s->work_size = 0; + } + if ((rc = drv(cu.cuMemAlloc(&s->work, need)))) goto out; + s->work_size = need; + } + + struct gcm_key key; + gcm_prepare(shim_sbox, p->key, p->nonce, p->aad, p->aad_length, n, &key); + /* Pageable source: the driver has copied `key` when this returns. */ + rc = drv(cu.cuMemcpyHtoDAsync(s->work + WORK_KEY, &key, sizeof(key), s->stream)); + wipe(&key, sizeof(key)); + if (rc) goto out; + s->decrypt = 1; + CUdeviceptr sealed = from->base + src_offset, out_ptr = d->base + dst_offset; + CUdeviceptr key_ptr = s->work + WORK_KEY, verdict = s->work + WORK_VERDICT; + CUdeviceptr partials = s->work + WORK_PARTIALS; + uint32_t check = p->check_crc; + uint64_t expected = p->expected_crc; + void *partial_args[] = {&sealed, &n, &key_ptr, &partials, &chunks}; + void *finalize_args[] = {&sealed, &n, &key_ptr, &partials, &chunks, &check, &expected, &verdict}; + void *ctr_args[] = {&sealed, &n, &key_ptr, &verdict, &out_ptr}; + uint64_t blocks = (n + 15) / 16; + rc = drv(cu.cuMemsetD8Async(verdict, 0, sizeof(struct gcm_verdict), s->stream)); + if (!rc) + rc = launch(s->context->partials, (chunks + PARTIAL_THREADS - 1) / PARTIAL_THREADS, + PARTIAL_THREADS, s->stream, partial_args); + if (!rc) rc = launch(s->context->finalize, 1, FINALIZE_THREADS, s->stream, finalize_args); + if (!rc && blocks) + rc = launch(s->context->ctr, (blocks + CTR_THREADS - 1) / CTR_THREADS, CTR_THREADS, s->stream, + ctr_args); + /* Always queue the wipe of key material and partials, even on failure. */ + int wiped = drv(cu.cuMemsetD8Async(key_ptr, 0, sizeof(struct gcm_key), s->stream)); + if (!wiped) + wiped = drv(cu.cuMemsetD8Async(partials, 0, chunks * sizeof(struct gcm_partial), s->stream)); + if (!rc) rc = wiped; + if (!rc) + rc = drv(cu.cuMemcpyDtoHAsync(s->verdict, verdict, sizeof(*s->verdict), s->stream)); +out: + pop(); + return rc; +} + +int racer_gpu_verdict(void *stream, struct racer_gpu_verdict *out) { + struct gpu_stream *s = stream; + if (!s || !out) return RACER_GPU_EINVAL; + if (s->decrypt != 2) return RACER_GPU_EBUSY; + *out = *s->verdict; + memset(s->verdict, 0, sizeof(*s->verdict)); + s->decrypt = 0; + return RACER_GPU_OK; +} + +static int file_error(CUfileError_t e) { + if (e.err == CU_FILE_SUCCESS) return RACER_GPU_OK; + last_error = e.err; + return RACER_GPU_EDRIVER; +} + +int racer_gpu_file_open(void) { + int rc = RACER_GPU_OK; + pthread_mutex_lock(&load_lock); + if (!cf.opened) { + struct cufile_table table; + memset(&table, 0, sizeof(table)); + table.lib = dlopen("libcufile.so.0", RTLD_NOW | RTLD_LOCAL); + if (!table.lib) { + rc = RACER_GPU_EUNAVAILABLE; + goto out; + } + RACER_CUFILE_FUNCTIONS(X_LOAD) + if (file_error(table.cuFileDriverOpen())) { + dlclose(table.lib); + rc = RACER_GPU_EUNAVAILABLE; + goto out; + } + /* Left open for the life of the process. */ + table.opened = 1; + cf = table; + goto out; + missing: + dlclose(table.lib); + rc = RACER_GPU_EUNAVAILABLE; + } +out: + pthread_mutex_unlock(&load_lock); + return rc; +} + +int racer_gpu_file(int fd, void **out) { + if (!cf.opened) return RACER_GPU_EUNAVAILABLE; + if (fd < 0 || !out) return RACER_GPU_EINVAL; + *out = NULL; + struct gpu_file *f = calloc(1, sizeof(*f)); + if (!f) return RACER_GPU_ENOMEM; + CUfileDescr_t descr; + memset(&descr, 0, sizeof(descr)); + descr.type = CU_FILE_HANDLE_TYPE_OPAQUE_FD; + descr.handle.fd = fd; + int rc = file_error(cf.cuFileHandleRegister(&f->handle, &descr)); + if (rc) { + free(f); + return rc; + } + *out = f; + return RACER_GPU_OK; +} + +int racer_gpu_file_free(void *file) { + struct gpu_file *f = file; + if (!f || !cf.opened) return RACER_GPU_EINVAL; + cf.cuFileHandleDeregister(f->handle); + free(f); + return RACER_GPU_OK; +} + +static int staging(void *p, struct gpu_staging **out) { + struct gpu_staging *b = p; + if (!b || b->b.kind != KIND_STAGING) return RACER_GPU_EINVAL; + *out = b; + return RACER_GPU_OK; +} + +int racer_gpu_staging_register(void *p) { + struct gpu_staging *b; + if (!cf.opened) return RACER_GPU_EUNAVAILABLE; + if (staging(p, &b)) return RACER_GPU_EINVAL; + if (b->registered) return RACER_GPU_EBUSY; + int rc = push(b->b.context); + if (rc) return rc; + rc = file_error(cf.cuFileBufRegister((const void *)(uintptr_t)b->b.base, b->allocated, 0)); + pop(); + if (!rc) b->registered = 1; + return rc; +} + +int racer_gpu_staging_unregister(void *p) { + struct gpu_staging *b; + if (!cf.opened || staging(p, &b)) return RACER_GPU_EINVAL; + if (!b->registered) return RACER_GPU_EBUSY; + int rc = push(b->b.context); + if (rc) return rc; + rc = file_error(cf.cuFileBufDeregister((const void *)(uintptr_t)b->b.base)); + pop(); + if (!rc) b->registered = 0; + return rc; +} + +int64_t racer_gpu_file_read(void *file, void *p, uint64_t offset, uint64_t file_offset, uint64_t length) { + struct gpu_file *f = file; + struct gpu_staging *b; + if (!cf.opened) return RACER_GPU_EUNAVAILABLE; + if (!f || staging(p, &b) || file_offset > (uint64_t)INT64_MAX - length) return RACER_GPU_EINVAL; + int rc = span(&b->b, offset, length); + if (rc) return rc; + if ((rc = push(b->b.context))) return rc; + ssize_t n = cf.cuFileRead(f->handle, (void *)(uintptr_t)b->b.base, length, (off_t)file_offset, + (off_t)offset); + pop(); + if (n < 0) { + last_error = (int32_t)n; + return RACER_GPU_EIO; + } + return (uint64_t)n == length ? (int64_t)n : RACER_GPU_EIO; +} + +int racer_gpu_dmabuf(void *p, uint64_t offset, uint64_t length, int *fd) { + struct gpu_staging *b; + long page = sysconf(_SC_PAGESIZE); + if (staging(p, &b) || !fd || page <= 0) return RACER_GPU_EINVAL; + if (!length || offset % (uint64_t)page || length % (uint64_t)page || offset > b->allocated || + length > b->allocated - offset) + return RACER_GPU_ERANGE; + *fd = -1; + int rc = push(b->b.context); + if (rc) return rc; + int handle = -1; + rc = drv(cu.cuMemGetHandleForAddressRange(&handle, b->b.base + offset, length, + CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, 0)); + pop(); + if (!rc) *fd = handle; + return rc; +} diff --git a/cmd/racer-dataplane/gpu/native/gpu_cuda.h b/cmd/racer-dataplane/gpu/native/gpu_cuda.h new file mode 100644 index 000000000..63c2f5a4f --- /dev/null +++ b/cmd/racer-dataplane/gpu/native/gpu_cuda.h @@ -0,0 +1,154 @@ +/* C ABI of libracer_gpu_cuda.so.1, loaded by racer-gpu (src/ffi.rs) with + * dlopen. The simulation backend (src/simulation.rs) implements the same + * functions in Rust. + * + * Bump RACER_GPU_ABI_VERSION whenever a struct, signature, or result + * contract changes. Rust refuses any other version. + * + * Every int result is RACER_GPU_OK or one of the negative codes below. A + * failed free leaves the resource in place and returns a code; Rust then + * leaks the resource on purpose because the GPU may still use it. Every call + * that takes an offset and length checks it against the buffer size, so the + * shim never touches memory outside a buffer. */ +#ifndef RACER_GPU_CUDA_H +#define RACER_GPU_CUDA_H + +#include +#include + +#define RACER_GPU_ABI_VERSION 1u + +#define RACER_GPU_OK 0 +#define RACER_GPU_EINVAL (-1) /* bad argument or handle type */ +#define RACER_GPU_ERANGE (-2) /* offset or length outside a buffer */ +#define RACER_GPU_EUNAVAILABLE (-3) /* driver library missing or init failed */ +#define RACER_GPU_EDEVICE (-4) /* allocation or buffer on another device */ +#define RACER_GPU_EDRIVER (-5) /* CUDA, NVML or cuFile call failed */ +#define RACER_GPU_ENOMEM (-6) /* host or device allocation failed */ +#define RACER_GPU_ENOKERNEL (-7) /* shim built without decrypt kernels */ +#define RACER_GPU_EIO (-8) /* cuFile read failed or was short */ +#define RACER_GPU_EBUSY (-9) /* wrong state, for example no verdict to read */ +#define RACER_GPU_ELEAKED (-10) /* failed, and cleanup leaked an internal resource */ + +/* Device capability bits in racer_gpu_device.flags. */ +#define RACER_GPU_FLAG_VMM 1u /* virtual memory management */ +#define RACER_GPU_FLAG_POSIX_FD 2u /* POSIX fd shareable handles */ +#define RACER_GPU_FLAG_DMA_BUF 4u /* dma-buf export */ +#define RACER_GPU_FLAG_GDR 8u /* GPUDirect RDMA */ + +/* CUcomputemode values (cuda.h). */ +#define RACER_GPU_COMPUTE_DEFAULT 0 +#define RACER_GPU_COMPUTE_PROHIBITED 2 +#define RACER_GPU_COMPUTE_EXCLUSIVE_PROCESS 3 + +/* One CUDA device. Mirrors `RawDevice` in ffi.rs. */ +struct racer_gpu_device { + uint8_t uuid[16]; + char pci_bus_id[32]; /* NUL terminated, for example "0000:3b:00.0" */ + int32_t ordinal; + int32_t numa_node; /* -1 when unknown */ + int32_t compute_mode; + uint32_t flags; + uint64_t total_memory; +}; + +/* One page to verify and decrypt. Mirrors `RawDecrypt` in ffi.rs. + * `key` and `nonce` are the XAES-derived AES-256-GCM key and nonce. The + * source holds `length` ciphertext bytes then the 16-byte tag. */ +struct racer_gpu_decrypt { + uint8_t key[32]; + uint8_t nonce[12]; + uint32_t check_crc; /* 1 to compare CRC-64/XZ with expected_crc */ + uint64_t expected_crc; + uint64_t length; + const uint8_t *aad; + uint64_t aad_length; +}; + +/* Mirrors `RawVerdict` in gcm.rs and `struct gcm_verdict`. */ +struct racer_gpu_verdict { + uint32_t status; + uint32_t reserved; + uint64_t crc; +}; + +_Static_assert(sizeof(struct racer_gpu_device) == 72, "device ABI size"); +_Static_assert(offsetof(struct racer_gpu_device, ordinal) == 48, "device ABI offset"); +_Static_assert(offsetof(struct racer_gpu_device, total_memory) == 64, "device ABI offset"); +_Static_assert(sizeof(struct racer_gpu_decrypt) == 80, "decrypt ABI size"); +_Static_assert(offsetof(struct racer_gpu_decrypt, check_crc) == 44, "decrypt ABI offset"); +_Static_assert(offsetof(struct racer_gpu_decrypt, aad) == 64, "decrypt ABI offset"); +_Static_assert(sizeof(struct racer_gpu_verdict) == 16, "verdict ABI size"); + +/* Largest page one decrypt call accepts (1 GiB). */ +#define RACER_GPU_MAX_DECRYPT (1ull << 30) +/* Largest AAD one decrypt call accepts. */ +#define RACER_GPU_MAX_AAD 4096u + +uint32_t racer_gpu_abi(void); +/* CUresult, NVML or cuFile status of this thread's last failed driver call. */ +int32_t racer_gpu_last_driver_error(void); +/* Load libcuda.so.1 and call cuInit. Safe to call more than once. */ +int racer_gpu_init(void); +/* Fill up to `capacity` devices. Returns the total count, which may exceed + * `capacity`, or a negative code. Reports every device, including ones in + * EXCLUSIVE_PROCESS or PROHIBITED mode; Rust decides which are usable. */ +int racer_gpu_discover(struct racer_gpu_device *out, uint32_t capacity); + +/* Retain the device's primary context. */ +int racer_gpu_context(int32_t ordinal, void **out); +int racer_gpu_context_free(void *context); + +int racer_gpu_stream(void *context, void **out); +int racer_gpu_stream_sync(void *stream); +/* Synchronizes, then frees the stream and its decrypt workspace. */ +int racer_gpu_stream_free(void *stream); + +/* Import a POSIX fd shareable handle of `size` bytes, check it lives on the + * context's device, map all of it and grant this context read-write access. + * The fd stays owned by the caller. */ +int racer_gpu_import(void *context, int fd, uint64_t size, void **out); +/* Unmap, free the address range and release the handle. The caller must + * have synchronized every stream that used it. */ +int racer_gpu_import_free(void *allocation); + +/* Allocate device staging memory, page aligned, usable with cuFile and + * dma-buf export. `size` is rounded up to 2 MiB. */ +int racer_gpu_staging(void *context, uint64_t size, void **out); +int racer_gpu_staging_free(void *staging); + +/* Queue a copy of host bytes into an import or staging buffer. The host + * buffer may be reused as soon as this returns. */ +int racer_gpu_copy_h2d(void *stream, void *dst, uint64_t offset, const void *src, uint64_t length); +/* Queue a device copy. `src` must be a staging buffer. */ +int racer_gpu_copy_d2d(void *stream, void *dst, uint64_t dst_offset, void *src, uint64_t src_offset, + uint64_t length); +/* Copy staging bytes to the host and wait. Imports are never read. */ +int racer_gpu_copy_d2h(void *stream, void *dst, void *src, uint64_t src_offset, uint64_t length); + +/* Queue verify-then-decrypt of the sealed page at `src + src_offset` (a + * staging buffer) into `dst + dst_offset`. The destination is written only if + * the CRC (when requested) and the tag match; the decision is made on the + * device. One decrypt per stream may be outstanding; read its verdict with + * racer_gpu_verdict after racer_gpu_stream_sync. */ +int racer_gpu_decrypt(void *stream, const struct racer_gpu_decrypt *params, void *src, + uint64_t src_offset, void *dst, uint64_t dst_offset); +int racer_gpu_verdict(void *stream, struct racer_gpu_verdict *out); + +/* Open the cuFile driver (libcufile.so.0). Safe to call more than once. */ +int racer_gpu_file_open(void); +/* Register an O_DIRECT fd with cuFile. The fd stays owned by the caller. */ +int racer_gpu_file(int fd, void **out); +int racer_gpu_file_free(void *file); +/* Register or deregister a staging buffer with cuFile. */ +int racer_gpu_staging_register(void *staging); +int racer_gpu_staging_unregister(void *staging); +/* Read `length` bytes at `file_offset` into staging at `offset`. Returns the + * byte count (always `length` on success) or a negative code. */ +int64_t racer_gpu_file_read(void *file, void *staging, uint64_t offset, uint64_t file_offset, + uint64_t length); +/* Export staging bytes as a dma-buf fd. Offset and length must be multiples + * of the host page size. */ +int racer_gpu_dmabuf(void *staging, uint64_t offset, uint64_t length, int *fd); + +#endif diff --git a/images/racer-dataplane/build-native-cuda.sh b/images/racer-dataplane/build-native-cuda.sh new file mode 100755 index 000000000..46a1bd971 --- /dev/null +++ b/images/racer-dataplane/build-native-cuda.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +# +# Build libracer_gpu_cuda.so.1 from cmd/racer-dataplane/gpu/native. +# +# The shim dlopens the CUDA driver, NVML and cuFile, so it needs no CUDA +# headers or libraries. When nvcc is found (NVCC, or nvcc on PATH) the decrypt +# kernels are compiled to PTX and embedded; otherwise the shim builds without +# them and GPU decryption reports "no kernels". Set RACER_GPU_REQUIRE_PTX=1 to +# fail instead of building without kernels. +set -eu + +if [ "$#" -ne 2 ]; then + echo "usage: build-native-cuda.sh NATIVE_DIR OUTPUT" >&2 + exit 2 +fi + +native=$1 +output=$2 +mkdir -p "$(dirname "$output")" +nvcc=${NVCC:-$(command -v nvcc || true)} +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +set -- +if [ -n "$nvcc" ]; then + # compute_80 (Ampere) is the PTX baseline; the driver JIT-compiles it for + # the installed GPU. + "$nvcc" -ptx -arch=compute_80 -O3 -std=c++14 -I "$native" \ + -o "$work/gcm_kernel.ptx" "$native/gcm_kernel.cu" + set -- "-DRACER_GPU_PTX=\"$work/gcm_kernel.ptx\"" +elif [ "${RACER_GPU_REQUIRE_PTX:-0}" = 1 ]; then + echo "nvcc not found and RACER_GPU_REQUIRE_PTX=1" >&2 + exit 1 +else + echo "nvcc not found: building libracer_gpu_cuda.so.1 without decrypt kernels" >&2 +fi + +"${CC:-cc}" "$@" -I "$native" \ + -O2 -g -fno-omit-frame-pointer -std=gnu11 -Wall -Wextra -Werror -fPIC -shared \ + -Wl,-soname,libracer_gpu_cuda.so.1 -Wl,-z,defs -Wl,-z,noexecstack \ + -o "$output" "$native/gpu_cuda.c" -ldl -lpthread From 3d7c60223da6d852f4a48d3f2c5a358518a76ca7 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 19:22:23 +0000 Subject: [PATCH 15/87] feat(racer): add racer-gpu safe API, shim loader, and simulation backend --- cmd/racer-dataplane/gpu/src/ffi.rs | 347 ++++++++ cmd/racer-dataplane/gpu/src/lib.rs | 862 ++++++++++++++++++- cmd/racer-dataplane/gpu/src/simulation.rs | 878 ++++++++++++++++++++ cmd/racer-dataplane/gpu/tests/simulation.rs | 445 ++++++++++ 4 files changed, 2531 insertions(+), 1 deletion(-) create mode 100644 cmd/racer-dataplane/gpu/src/ffi.rs create mode 100644 cmd/racer-dataplane/gpu/src/simulation.rs create mode 100644 cmd/racer-dataplane/gpu/tests/simulation.rs diff --git a/cmd/racer-dataplane/gpu/src/ffi.rs b/cmd/racer-dataplane/gpu/src/ffi.rs new file mode 100644 index 000000000..000be6e0f --- /dev/null +++ b/cmd/racer-dataplane/gpu/src/ffi.rs @@ -0,0 +1,347 @@ +//! The C ABI of `libracer_gpu_cuda.so.1` (`native/gpu_cuda.h`) and its loader. +//! +//! Every safe type in `lib.rs` holds an `Rc`, which holds the [`Api`] +//! symbol table, so the library stays loaded while any GPU object lives. The +//! simulation backend builds the same table from Rust functions. +use crate::{Error, Result}; + +use std::{ + cell::Cell, + ffi::{c_int, c_void}, + rc::Rc, +}; + +pub(crate) use crate::gcm::RawVerdict; + +/// `RACER_GPU_ABI_VERSION`. +#[cfg(all(feature = "native", target_os = "linux"))] +pub(crate) const ABI_VERSION: u32 = 1; + +/// Result codes from `gpu_cuda.h`. +pub(crate) mod code { + use std::ffi::c_int; + + pub const OK: c_int = 0; + pub const EINVAL: c_int = -1; + pub const ERANGE: c_int = -2; + pub const EUNAVAILABLE: c_int = -3; + pub const EDEVICE: c_int = -4; + pub const EDRIVER: c_int = -5; + pub const ENOMEM: c_int = -6; + pub const ENOKERNEL: c_int = -7; + pub const EIO: c_int = -8; + pub const EBUSY: c_int = -9; + pub const ELEAKED: c_int = -10; +} + +/// Device capability bits (`RACER_GPU_FLAG_*`). +pub mod flag { + /// Virtual memory management (import and map). + pub const VMM: u32 = 1; + /// POSIX fd shareable handles. + pub const POSIX_FD: u32 = 2; + /// dma-buf export. + pub const DMA_BUF: u32 = 4; + /// GPUDirect RDMA. + pub const GDR: u32 = 8; +} + +/// One device. Matches `struct racer_gpu_device`. +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct RawDevice { + pub uuid: [u8; 16], + + pub pci_bus_id: [u8; 32], + + pub ordinal: i32, + + pub numa_node: i32, + + pub compute_mode: i32, + + pub flags: u32, + + pub total_memory: u64, +} + +impl Default for RawDevice { + /// Unknown NUMA node and compute mode. + fn default() -> Self { + Self { + uuid: [0; 16], + pci_bus_id: [0; 32], + ordinal: 0, + numa_node: -1, + compute_mode: -1, + flags: 0, + total_memory: 0, + } + } +} + +/// One decrypt request. Matches `struct racer_gpu_decrypt`. Holds a key, so +/// callers wipe it after the call. +#[repr(C)] +pub(crate) struct RawDecrypt { + pub key: [u8; 32], + + pub nonce: [u8; 12], + + pub check_crc: u32, + + pub expected_crc: u64, + + pub length: u64, + + pub aad: *const u8, + + pub aad_length: u64, +} + +impl Drop for RawDecrypt { + /// Wipe the key and nonce. + fn drop(&mut self) { + use zeroize::Zeroize; + self.key.zeroize(); + self.nonce.zeroize(); + } +} + +/// `RACER_GPU_MAX_DECRYPT`. +pub const MAX_DECRYPT: u64 = 1 << 30; + +/// `RACER_GPU_MAX_AAD`. +pub const MAX_AAD: usize = 4096; + +/// Opaque handle out-parameter. +pub(crate) type Out = *mut *mut c_void; + +/// Build [`Api`] and its symbol loader from `racer_gpu_` functions. +macro_rules! native_api { + ($($field:ident: $signature:ty),* $(,)?) => { + /// The shim's functions. + pub(crate) struct Api { + /// `dlopen` handle; `None` for the simulation. + #[cfg_attr(not(all(feature = "native", target_os = "linux")), allow(dead_code))] + pub(crate) library: Option>, + + $( + pub(crate) $field: $signature, + + )* + } + + impl Api { + #[cfg(all(feature = "native", target_os = "linux"))] + /// Look up every function in `library`. + unsafe fn symbols(library: std::ptr::NonNull) -> Result { + Ok(Self { + library: Some(library), + $($field: { + let pointer = unsafe { libc::dlsym(library.as_ptr(), + concat!("racer_gpu_", stringify!($field), "\0").as_ptr().cast()) }; + if pointer.is_null() { return Err(Error::Unavailable); } + unsafe { std::mem::transmute::<*mut c_void, $signature>(pointer) } + },)* + }) + } + } + }; +} + +native_api! { + last_driver_error: unsafe extern "C" fn() -> i32, + + init: unsafe extern "C" fn() -> c_int, + + discover: unsafe extern "C" fn(*mut RawDevice, u32) -> c_int, + + context: unsafe extern "C" fn(i32, Out) -> c_int, + + context_free: unsafe extern "C" fn(*mut c_void) -> c_int, + + stream: unsafe extern "C" fn(*mut c_void, Out) -> c_int, + + stream_sync: unsafe extern "C" fn(*mut c_void) -> c_int, + + stream_free: unsafe extern "C" fn(*mut c_void) -> c_int, + + import: unsafe extern "C" fn(*mut c_void, c_int, u64, Out) -> c_int, + + import_free: unsafe extern "C" fn(*mut c_void) -> c_int, + + staging: unsafe extern "C" fn(*mut c_void, u64, Out) -> c_int, + + staging_free: unsafe extern "C" fn(*mut c_void) -> c_int, + + copy_h2d: unsafe extern "C" fn(*mut c_void, *mut c_void, u64, *const c_void, u64) -> c_int, + + copy_d2d: unsafe extern "C" fn(*mut c_void, *mut c_void, u64, *mut c_void, u64, u64) -> c_int, + + copy_d2h: unsafe extern "C" fn(*mut c_void, *mut c_void, *mut c_void, u64, u64) -> c_int, + + decrypt: unsafe extern "C" fn( + *mut c_void, + *const RawDecrypt, + *mut c_void, + u64, + *mut c_void, + u64, + ) -> c_int, + + verdict: unsafe extern "C" fn(*mut c_void, *mut RawVerdict) -> c_int, + + file_open: unsafe extern "C" fn() -> c_int, + + file: unsafe extern "C" fn(c_int, Out) -> c_int, + + file_free: unsafe extern "C" fn(*mut c_void) -> c_int, + + staging_register: unsafe extern "C" fn(*mut c_void) -> c_int, + + staging_unregister: unsafe extern "C" fn(*mut c_void) -> c_int, + + file_read: unsafe extern "C" fn(*mut c_void, *mut c_void, u64, u64, u64) -> i64, + + dmabuf: unsafe extern "C" fn(*mut c_void, u64, u64, *mut c_int) -> c_int, +} + +impl Api { + #[cfg(not(all(feature = "native", target_os = "linux")))] + /// No native backend in this build. + pub(crate) fn load() -> Result> { + Err(Error::Unavailable) + } + + #[cfg(all(feature = "native", target_os = "linux"))] + /// Load `libracer_gpu_cuda.so.1` and check its ABI version. Does not + /// initialize CUDA. + pub(crate) fn load() -> Result> { + use std::ptr::NonNull; + + unsafe { + let library = NonNull::new(libc::dlopen( + c"libracer_gpu_cuda.so.1".as_ptr(), + libc::RTLD_NOW | libc::RTLD_LOCAL, + )) + .ok_or(Error::Unavailable)?; + + /// Unloads the library if we bail out early. + struct Guard(NonNull); + + impl Drop for Guard { + /// Unload. + fn drop(&mut self) { + unsafe { + libc::dlclose(self.0.as_ptr()); + } + } + } + + let guard = Guard(library); + let version = libc::dlsym(library.as_ptr(), c"racer_gpu_abi".as_ptr()); + if version.is_null() { + return Err(Error::Unavailable); + } + let version = + std::mem::transmute::<*mut c_void, unsafe extern "C" fn() -> u32>(version); + if version() != ABI_VERSION { + return Err(Error::Unavailable); + } + let api = Self::symbols(library)?; + std::mem::forget(guard); + Ok(Rc::new(api)) + } + } +} + +impl Drop for Api { + /// Unload the library. Leaked objects keep it loaded forever. + fn drop(&mut self) { + #[cfg(all(feature = "native", target_os = "linux"))] + if let Some(library) = self.library { + unsafe { + libc::dlclose(library.as_ptr()); + } + } + } +} + +/// The loaded backend and the leak counter every object shares. +pub(crate) struct Shared { + pub(crate) api: Rc, + + pub(crate) leaks: Cell, +} + +impl Shared { + /// Map a shim result code to an error. + pub(crate) fn check(&self, rc: c_int) -> Result<()> { + match rc { + code::OK => Ok(()), + code::EINVAL => Err(Error::InvalidRequest), + code::ERANGE => Err(Error::InvalidRange), + code::EUNAVAILABLE => Err(Error::Unavailable), + code::EDEVICE => Err(Error::WrongDevice), + code::EDRIVER => Err(Error::Driver(unsafe { (self.api.last_driver_error)() })), + code::ENOMEM => Err(Error::OutOfMemory), + code::ENOKERNEL => Err(Error::KernelUnavailable), + code::EIO => Err(Error::Io), + code::EBUSY => Err(Error::Busy), + code::ELEAKED => { + self.leaks.set(self.leaks.get() + 1); + Err(Error::Leaked) + } + _ => Err(Error::Driver(rc)), + } + } + + /// Count a leaked resource and keep the library loaded for good. + pub(crate) fn leak(self: &Rc) { + self.leaks.set(self.leaks.get() + 1); + std::mem::forget(self.clone()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + /// Rust ABI records match the shim's `_Static_assert`s in gpu_cuda.h. + fn abi_layout_matches_c_static_assertions() { + assert_eq!(std::mem::size_of::(), 72); + assert_eq!(std::mem::offset_of!(RawDevice, ordinal), 48); + assert_eq!(std::mem::offset_of!(RawDevice, total_memory), 64); + assert_eq!(std::mem::size_of::(), 80); + assert_eq!(std::mem::offset_of!(RawDecrypt, check_crc), 44); + assert_eq!(std::mem::offset_of!(RawDecrypt, aad), 64); + assert_eq!(std::mem::size_of::(), 16); + } + + #[cfg(all(feature = "native", target_os = "linux"))] + #[test] + #[ignore = "requires libracer_gpu_cuda.so.1 on the loader path"] + /// The installed shim loads, speaks this ABI, and exports every symbol. + /// Without a CUDA driver, every entry point fails cleanly. + fn required_native_shim_loads_abi_and_symbols() { + let api = Api::load().expect("libracer_gpu_cuda.so.1 must be installed for this test"); + assert!(api.library.is_some()); + let rc = unsafe { (api.init)() }; + assert!( + rc == code::OK || rc == code::EUNAVAILABLE, + "init returned {rc}" + ); + if rc == code::EUNAVAILABLE { + let mut device = RawDevice::default(); + assert_eq!( + unsafe { (api.discover)(&mut device, 1) }, + code::EUNAVAILABLE + ); + let mut out = std::ptr::null_mut(); + assert_eq!(unsafe { (api.context)(0, &mut out) }, code::EUNAVAILABLE); + assert_eq!(unsafe { (api.file)(3, &mut out) }, code::EUNAVAILABLE); + } + } +} diff --git a/cmd/racer-dataplane/gpu/src/lib.rs b/cmd/racer-dataplane/gpu/src/lib.rs index 73a682323..5c4949fb5 100644 --- a/cmd/racer-dataplane/gpu/src/lib.rs +++ b/cmd/racer-dataplane/gpu/src/lib.rs @@ -1,2 +1,862 @@ -//! GPU support for Racer (placeholder while the crate is assembled). +//! CUDA support for Racer GPUDirect reads (`designs/racer-gpudirect.md`). +//! +//! Every CUDA, NVML and cuFile call goes through a small C shim, +//! `libracer_gpu_cuda.so.1` (`native/gpu_cuda.c`), loaded with `dlopen` and +//! checked with `racer_gpu_abi()`. All types here are `!Send`: build and use +//! them on the dataplane's native helper thread, never on I/O threads. +//! +//! # Basics +//! +//! 1. [`Gpu::load`] loads the backend and initializes CUDA. +//! 2. [`Gpu::devices`] lists GPUs; [`DeviceInfo::usable`] skips GPUs in +//! `EXCLUSIVE_PROCESS` or `PROHIBITED` compute mode. +//! 3. [`Gpu::context`] retains a GPU's primary context. +//! 4. [`Context::import`] maps a client allocation (a POSIX fd shareable +//! handle) for one request. Writes are limited to the request's window. +//! 5. [`Context::staging`] allocates dataplane-owned device memory for +//! ciphertext. It can be filled from the host, from disk with cuFile +//! ([`GpuFile::read`]) or by a NIC through a dma-buf ([`StagingBuffer::dma_buf`]). +//! 6. [`Stream`] queues copies and [`Stream::decrypt`], then +//! [`Stream::synchronize`] waits for them. Only then is the write visible. +//! +//! # Lifetimes and leaks +//! +//! A stream keeps every buffer it touched alive until a successful +//! synchronize, so an import is never unmapped while queued GPU work may still +//! write it. If a free, unmap or synchronize fails, the resource is leaked on +//! purpose, counted in [`Gpu::leaked`], and the shim stays loaded. +//! +//! # Backends +//! +//! - `native`: loads `libracer_gpu_cuda.so.1`. +//! - `simulation`: [`simulation::Simulation`], fake GPUs in host memory. +//! - Neither: [`Gpu::load`] returns [`Error::Unavailable`]. +//! +//! Hardware-free tests: `cargo test -p racer-gpu --features simulation`. +mod ffi; pub mod gcm; + +#[cfg(any(test, feature = "simulation"))] +pub mod simulation; + +pub use ffi::{MAX_AAD, MAX_DECRYPT, flag}; + +use ffi::{Out, RawDecrypt, RawDevice, RawVerdict, Shared}; +use std::{ + cell::{Cell, RefCell}, + ffi::c_void, + ops::Range, + os::fd::{AsRawFd, FromRawFd, OwnedFd}, + ptr::NonNull, + rc::Rc, +}; + +/// Result type for this crate. +pub type Result = std::result::Result; + +/// Errors from this crate. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Error { + /// No backend, no driver, or the feature is not built in. + Unavailable, + + /// Bad arguments or handle. + InvalidRequest, + + /// An offset or length is outside a buffer or the request window. + InvalidRange, + + /// The allocation or buffer is on another GPU. + WrongDevice, + + /// A CUDA, NVML or cuFile call failed with this status. + Driver(i32), + + /// Host or device memory allocation failed. + OutOfMemory, + + /// The shim was built without decrypt kernels, or they failed to load. + KernelUnavailable, + + /// A cuFile read failed or was short. + Io, + + /// Wrong state: work in flight, or no verdict to read. + Busy, + + /// The call failed and a resource was leaked on purpose. + Leaked, +} + +impl std::fmt::Display for Error { + /// Describe the error. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Driver(code) => write!(f, "GPU driver call failed with status {code}"), + other => write!(f, "{other:?}"), + } + } +} + +impl std::error::Error for Error {} + +/// A GPU's compute mode (`CUcomputemode`). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ComputeMode { + /// Several contexts may share the GPU. + Default, + + /// No contexts allowed. + Prohibited, + + /// One process at a time. + ExclusiveProcess, + + /// Unknown, or the query failed. + Other(i32), +} + +impl ComputeMode { + /// From the raw `CUcomputemode` value. + fn from_raw(value: i32) -> Self { + match value { + 0 => Self::Default, + 2 => Self::Prohibited, + 3 => Self::ExclusiveProcess, + other => Self::Other(other), + } + } +} + +/// One GPU from discovery. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DeviceInfo { + /// CUDA device ordinal. + pub ordinal: i32, + + /// GPU UUID. + pub uuid: [u8; 16], + + /// Lowercase PCI address, for example `0000:3b:00.0`. + pub pci_bus_id: String, + + /// NUMA node from NVML or sysfs, if known. + pub numa_node: Option, + + /// Compute mode. + pub compute_mode: ComputeMode, + + /// `flag::*` capability bits. + pub flags: u32, + + /// Device memory in bytes. + pub total_memory: u64, +} + +impl DeviceInfo { + /// Whether the dataplane can use this GPU: default compute mode, and it + /// can import POSIX fd handles into virtual memory. + pub fn usable(&self) -> bool { + self.compute_mode == ComputeMode::Default + && self.flags & (flag::VMM | flag::POSIX_FD) == flag::VMM | flag::POSIX_FD + } + + /// The UUID in `GPU-xxxxxxxx-xxxx-...` form, as `nvidia-smi` prints it. + pub fn uuid_string(&self) -> String { + let hex: String = self.uuid.iter().map(|b| format!("{b:02x}")).collect(); + format!( + "GPU-{}-{}-{}-{}-{}", + &hex[..8], + &hex[8..12], + &hex[12..16], + &hex[16..20], + &hex[20..] + ) + } + + /// From the C record. + fn from_raw(raw: &RawDevice) -> Self { + let end = raw.pci_bus_id.iter().position(|&b| b == 0).unwrap_or(32); + Self { + ordinal: raw.ordinal, + uuid: raw.uuid, + pci_bus_id: String::from_utf8_lossy(&raw.pci_bus_id[..end]).into_owned(), + numa_node: u32::try_from(raw.numa_node).ok(), + compute_mode: ComputeMode::from_raw(raw.compute_mode), + flags: raw.flags, + total_memory: raw.total_memory, + } + } +} + +/// Call a shim function that writes a handle; return it or the error. +fn create(shared: &Shared, call: impl FnOnce(Out) -> i32) -> Result> { + let mut out = std::ptr::null_mut(); + shared.check(call(&mut out))?; + NonNull::new(out).ok_or(Error::InvalidRequest) +} + +/// A loaded GPU backend with CUDA initialized. +pub struct Gpu { + shared: Rc, +} + +impl Gpu { + /// Load the backend and initialize CUDA. Inside a + /// [`simulation::Simulation::enter`] scope this uses the simulation. + pub fn load() -> Result { + #[cfg(any(test, feature = "simulation"))] + let api = match simulation::current() { + Some(_) => simulation::api(), + None => ffi::Api::load()?, + }; + #[cfg(not(any(test, feature = "simulation")))] + let api = ffi::Api::load()?; + let shared = Rc::new(Shared { + api, + leaks: Cell::new(0), + }); + shared.check(unsafe { (shared.api.init)() })?; + Ok(Self { shared }) + } + + /// Every GPU, including ones that are not [`DeviceInfo::usable`]. + pub fn devices(&self) -> Result> { + let mut raw = vec![RawDevice::default(); 64]; + let count = unsafe { (self.shared.api.discover)(raw.as_mut_ptr(), raw.len() as u32) }; + self.shared.check(count.min(0))?; + let count = count as usize; + if count > raw.len() { + return Err(Error::InvalidRequest); + } + Ok(raw[..count].iter().map(DeviceInfo::from_raw).collect()) + } + + /// The usable GPUs only. + pub fn usable_devices(&self) -> Result> { + Ok(self + .devices()? + .into_iter() + .filter(DeviceInfo::usable) + .collect()) + } + + /// Retain a GPU's primary context. + pub fn context(&self, ordinal: i32) -> Result { + let shared = &self.shared; + let raw = create(shared, |out| unsafe { (shared.api.context)(ordinal, out) })?; + Ok(Context(Rc::new(ContextInner { + shared: shared.clone(), + raw, + ordinal, + }))) + } + + /// Open the cuFile driver if needed and register an `O_DIRECT` file. + pub fn open_file(&self, fd: OwnedFd) -> Result { + let shared = &self.shared; + shared.check(unsafe { (shared.api.file_open)() })?; + let raw = create(shared, |out| unsafe { + (shared.api.file)(fd.as_raw_fd(), out) + })?; + Ok(GpuFile { + shared: shared.clone(), + raw, + fd: Some(fd), + }) + } + + /// Resources leaked on purpose since this backend was loaded. + pub fn leaked(&self) -> u64 { + self.shared.leaks.get() + } +} + +/// A retained primary context. Clones share it; it is released when the last +/// clone and every object made from it are gone. +#[derive(Clone)] +pub struct Context(Rc); + +/// The owned context handle. +struct ContextInner { + shared: Rc, + + raw: NonNull, + + ordinal: i32, +} + +impl Drop for ContextInner { + /// Release the context, or leak it if that fails. + fn drop(&mut self) { + if unsafe { (self.shared.api.context_free)(self.raw.as_ptr()) } != 0 { + self.shared.leak(); + } + } +} + +impl Context { + /// The CUDA device ordinal. + pub fn ordinal(&self) -> i32 { + self.0.ordinal + } + + /// Leaks seen by this context's backend. + pub fn leaked(&self) -> u64 { + self.0.shared.leaks.get() + } + + /// Create a non-blocking stream. + pub fn stream(&self) -> Result { + let shared = &self.0.shared; + let raw = create(shared, |out| unsafe { + (shared.api.stream)(self.0.raw.as_ptr(), out) + })?; + Ok(Stream { + context: self.clone(), + raw, + retained: RefCell::new(Vec::new()), + }) + } + + /// Import a client allocation and map all `allocation_size` bytes + /// read-write. Writes are then allowed only inside `window`, the + /// request's `[offset, offset + length)`. Keeps `fd` until the import is + /// released, then closes it. + pub fn import( + &self, + fd: OwnedFd, + allocation_size: u64, + window: Range, + ) -> Result { + if window.start >= window.end || window.end > allocation_size { + return Err(Error::InvalidRange); + } + let shared = &self.0.shared; + let raw = create(shared, |out| unsafe { + (shared.api.import)(self.0.raw.as_ptr(), fd.as_raw_fd(), allocation_size, out) + })?; + Ok(ImportedAllocation(Rc::new(ImportInner { + context: self.clone(), + raw, + size: allocation_size, + window, + fd: Cell::new(Some(fd)), + }))) + } + + /// Allocate `size` bytes of staging memory (rounded up to 2 MiB on the + /// device; offsets are checked against `size`). + pub fn staging(&self, size: u64) -> Result { + let shared = &self.0.shared; + let raw = create(shared, |out| unsafe { + (shared.api.staging)(self.0.raw.as_ptr(), size, out) + })?; + Ok(StagingBuffer(Rc::new(StagingInner { + context: self.clone(), + raw, + size, + registered: Cell::new(false), + in_flight: Cell::new(0), + }))) + } +} + +/// A client allocation mapped into the dataplane's context for one request. +pub struct ImportedAllocation(Rc); + +/// The mapping. Freed when the handle and every stream using it let go. +struct ImportInner { + context: Context, + + raw: NonNull, + + size: u64, + + window: Range, + + fd: Cell>, +} + +impl Drop for ImportInner { + /// Unmap and release, then close the fd. On failure leak everything, + /// including the fd and the context. + fn drop(&mut self) { + let shared = &self.context.0.shared; + if unsafe { (shared.api.import_free)(self.raw.as_ptr()) } != 0 { + shared.leak(); + std::mem::forget(self.fd.take()); + std::mem::forget(self.context.clone()); + } + } +} + +impl ImportedAllocation { + /// The size passed to `cuMemCreate`. + pub fn size(&self) -> u64 { + self.0.size + } + + /// The writable window. + pub fn window(&self) -> Range { + self.0.window.clone() + } + + /// The GPU it lives on. + pub fn ordinal(&self) -> i32 { + self.0.context.ordinal() + } +} + +/// Dataplane-owned device memory for ciphertext. +pub struct StagingBuffer(Rc); + +/// The allocation. +struct StagingInner { + context: Context, + + raw: NonNull, + + size: u64, + + registered: Cell, + + /// Streams holding it until their next synchronize. + in_flight: Cell, +} + +impl Drop for StagingInner { + /// Deregister from cuFile, then free. On failure leak. + fn drop(&mut self) { + let shared = &self.context.0.shared; + if self.registered.get() + && unsafe { (shared.api.staging_unregister)(self.raw.as_ptr()) } != 0 + { + shared.leak(); + std::mem::forget(self.context.clone()); + return; + } + if unsafe { (shared.api.staging_free)(self.raw.as_ptr()) } != 0 { + shared.leak(); + std::mem::forget(self.context.clone()); + } + } +} + +impl StagingBuffer { + /// Usable bytes. + pub fn size(&self) -> u64 { + self.0.size + } + + /// Whether a stream may still be using it. + pub fn in_flight(&self) -> bool { + self.0.in_flight.get() > 0 + } + + /// Register with cuFile so reads go straight to it. + pub fn register_cufile(&self) -> Result<()> { + if self.0.registered.get() { + return Ok(()); + } + let shared = &self.0.context.0.shared; + shared.check(unsafe { (shared.api.file_open)() })?; + shared.check(unsafe { (shared.api.staging_register)(self.0.raw.as_ptr()) })?; + self.0.registered.set(true); + Ok(()) + } + + /// Export `[offset, offset + length)` as a dma-buf fd, for + /// `ibv_reg_dmabuf_mr`. Both must be multiples of the host page size. + pub fn dma_buf(&self, offset: u64, length: u64) -> Result { + let shared = &self.0.context.0.shared; + let mut fd = -1; + shared + .check(unsafe { (shared.api.dmabuf)(self.0.raw.as_ptr(), offset, length, &mut fd) })?; + if fd < 0 { + return Err(Error::InvalidRequest); + } + Ok(unsafe { OwnedFd::from_raw_fd(fd) }) + } +} + +/// Where a stream writes. +#[derive(Clone, Copy)] +pub enum Destination<'a> { + /// A client allocation, limited to its window. + Import(&'a ImportedAllocation), + + /// Dataplane staging memory. + Staging(&'a StagingBuffer), +} + +/// Something a stream keeps alive until synchronized. +enum Retained { + Import(#[allow(dead_code)] Rc), + + Staging(Rc), +} + +impl Drop for Retained { + /// Release one in-flight hold on staging memory. + fn drop(&mut self) { + if let Self::Staging(inner) = self { + inner.in_flight.set(inner.in_flight.get() - 1); + } + } +} + +/// One page to verify and decrypt on the GPU. +pub struct DecryptRequest<'a> { + /// XAES-derived AES-256-GCM key (`racer_crypto::DerivedPageKey::key`). + pub key: &'a [u8; 32], + + /// XAES-derived GCM nonce (`racer_crypto::DerivedPageKey::nonce`). + pub nonce: &'a [u8; 12], + + /// The page AAD (`security::page_aad`). At most [`MAX_AAD`] bytes. + pub aad: &'a [u8], + + /// Plaintext length. The source holds this many bytes plus the tag. + pub length: u64, + + /// Stored CRC-64/XZ of the sealed page, if known. + pub expected_crc: Option, +} + +/// The outcome of a GPU decrypt. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Verdict { + /// Verified and written. `crc` is the sealed page's CRC-64/XZ. + Passed { + /// CRC-64/XZ of the sealed page. + crc: u64, + }, + + /// The CRC did not match. Nothing was written. + CrcMismatch { + /// The CRC the GPU computed. + crc: u64, + }, + + /// The tag did not match. Nothing was written. + TagMismatch { + /// The CRC the GPU computed. + crc: u64, + }, + + /// Verification did not run (a launch failed). Nothing was written. + NotRun, +} + +/// A CUDA stream. Work is visible only after [`Stream::synchronize`]. +pub struct Stream { + context: Context, + + raw: NonNull, + + retained: RefCell>, +} + +impl Stream { + /// The backend state. + fn shared(&self) -> &Rc { + &self.context.0.shared + } + + /// Check a destination range and return its raw handle and hold. + fn destination( + &self, + dst: Destination<'_>, + offset: u64, + length: u64, + ) -> Result<(*mut c_void, Retained)> { + let end = offset.checked_add(length).ok_or(Error::InvalidRange)?; + match dst { + Destination::Import(import) => { + let inner = &import.0; + if !Rc::ptr_eq(&inner.context.0, &self.context.0) { + return Err(Error::WrongDevice); + } + if offset < inner.window.start || end > inner.window.end { + return Err(Error::InvalidRange); + } + Ok((inner.raw.as_ptr(), Retained::Import(inner.clone()))) + } + Destination::Staging(staging) => self.source(staging, offset, length), + } + } + + /// Check a staging range and return its raw handle and hold. + fn source( + &self, + staging: &StagingBuffer, + offset: u64, + length: u64, + ) -> Result<(*mut c_void, Retained)> { + let inner = &staging.0; + if !Rc::ptr_eq(&inner.context.0, &self.context.0) { + return Err(Error::WrongDevice); + } + let end = offset.checked_add(length).ok_or(Error::InvalidRange)?; + if end > inner.size { + return Err(Error::InvalidRange); + } + inner.in_flight.set(inner.in_flight.get() + 1); + Ok((inner.raw.as_ptr(), Retained::Staging(inner.clone()))) + } + + /// Keep holds until the next synchronize. Held even when the call + /// failed, because part of it may have been queued. + fn retain(&self, holds: impl IntoIterator) { + self.retained.borrow_mut().extend(holds); + } + + /// Queue a copy of host bytes. `bytes` may be reused once this returns. + pub fn copy_from_host(&self, dst: Destination<'_>, offset: u64, bytes: &[u8]) -> Result<()> { + if bytes.is_empty() { + return Err(Error::InvalidRange); + } + let (raw, hold) = self.destination(dst, offset, bytes.len() as u64)?; + let rc = unsafe { + (self.shared().api.copy_h2d)( + self.raw.as_ptr(), + raw, + offset, + bytes.as_ptr().cast(), + bytes.len() as u64, + ) + }; + self.retain([hold]); + self.shared().check(rc) + } + + /// Queue a device copy from staging memory. + pub fn copy_from_staging( + &self, + dst: Destination<'_>, + dst_offset: u64, + src: &StagingBuffer, + src_offset: u64, + length: u64, + ) -> Result<()> { + if length == 0 { + return Err(Error::InvalidRange); + } + let (raw_dst, dst_hold) = self.destination(dst, dst_offset, length)?; + let (raw_src, src_hold) = self.source(src, src_offset, length)?; + let rc = unsafe { + (self.shared().api.copy_d2d)( + self.raw.as_ptr(), + raw_dst, + dst_offset, + raw_src, + src_offset, + length, + ) + }; + self.retain([dst_hold, src_hold]); + self.shared().check(rc) + } + + /// Copy staging bytes to the host and wait for this stream. Client + /// imports can never be read. + pub fn read_staging(&self, src: &StagingBuffer, offset: u64, out: &mut [u8]) -> Result<()> { + if out.is_empty() { + return Err(Error::InvalidRange); + } + let (raw, hold) = self.source(src, offset, out.len() as u64)?; + let rc = unsafe { + (self.shared().api.copy_d2h)( + self.raw.as_ptr(), + out.as_mut_ptr().cast(), + raw, + offset, + out.len() as u64, + ) + }; + self.retain([hold]); + self.shared().check(rc)?; + // copy_d2h synchronized the stream. + self.retained.borrow_mut().clear(); + Ok(()) + } + + /// Queue verify-then-decrypt of the sealed page at `src_offset` into + /// `dst` at `dst_offset`. The destination is written only if the checks + /// pass, decided on the GPU. Read the result with [`Stream::verdict`] + /// after [`Stream::synchronize`]. One decrypt per synchronize; a + /// second one first returns [`Error::Busy`]. + pub fn decrypt( + &self, + request: &DecryptRequest<'_>, + src: &StagingBuffer, + src_offset: u64, + dst: Destination<'_>, + dst_offset: u64, + ) -> Result<()> { + if request.length > MAX_DECRYPT || request.aad.len() > MAX_AAD { + return Err(Error::InvalidRequest); + } + let mut holds = Vec::with_capacity(2); + let (raw_src, hold) = self.source(src, src_offset, request.length + 16)?; + holds.push(hold); + // A zero-length page writes nothing but still needs a destination + // on this GPU inside the window. + let (raw_dst, hold) = self.destination(dst, dst_offset, request.length)?; + holds.push(hold); + let raw = RawDecrypt { + key: *request.key, + nonce: *request.nonce, + check_crc: request.expected_crc.is_some() as u32, + expected_crc: request.expected_crc.unwrap_or(0), + length: request.length, + aad: request.aad.as_ptr(), + aad_length: request.aad.len() as u64, + }; + let rc = unsafe { + (self.shared().api.decrypt)( + self.raw.as_ptr(), + &raw, + raw_src, + src_offset, + raw_dst, + dst_offset, + ) + }; + drop(raw); + self.retain(holds); + self.shared().check(rc) + } + + /// Wait for all queued work, then release holds on the buffers it used. + pub fn synchronize(&self) -> Result<()> { + self.shared() + .check(unsafe { (self.shared().api.stream_sync)(self.raw.as_ptr()) })?; + self.retained.borrow_mut().clear(); + Ok(()) + } + + /// The verdict of the last decrypt, after [`Stream::synchronize`]. + pub fn verdict(&self) -> Result { + let mut raw = RawVerdict::default(); + self.shared() + .check(unsafe { (self.shared().api.verdict)(self.raw.as_ptr(), &mut raw) })?; + Ok(match raw.status { + gcm::VERDICT_PASS => Verdict::Passed { crc: raw.crc }, + gcm::VERDICT_CRC => Verdict::CrcMismatch { crc: raw.crc }, + gcm::VERDICT_TAG => Verdict::TagMismatch { crc: raw.crc }, + _ => Verdict::NotRun, + }) + } +} + +impl Drop for Stream { + /// Synchronize and free. If that fails, leak the stream and every buffer + /// it may still be using. + fn drop(&mut self) { + let shared = self.context.0.shared.clone(); + if unsafe { (shared.api.stream_free)(self.raw.as_ptr()) } != 0 { + shared.leak(); + std::mem::forget(std::mem::take(&mut *self.retained.borrow_mut())); + std::mem::forget(self.context.clone()); + } + } +} + +/// A file registered with cuFile. +pub struct GpuFile { + shared: Rc, + + raw: NonNull, + + fd: Option, +} + +impl GpuFile { + /// Read `length` bytes at `file_offset` into staging at `offset`. Waits + /// for the read. The staging buffer must not be in use by a stream. + pub fn read( + &self, + staging: &StagingBuffer, + offset: u64, + file_offset: u64, + length: u64, + ) -> Result<()> { + if staging.in_flight() { + return Err(Error::Busy); + } + let end = offset.checked_add(length).ok_or(Error::InvalidRange)?; + if length == 0 || end > staging.size() { + return Err(Error::InvalidRange); + } + let n = unsafe { + (self.shared.api.file_read)( + self.raw.as_ptr(), + staging.0.raw.as_ptr(), + offset, + file_offset, + length, + ) + }; + if n < 0 { + return self.shared.check(n as i32); + } + if n as u64 != length { + return Err(Error::Io); + } + Ok(()) + } +} + +impl Drop for GpuFile { + /// Deregister, then close the fd. On failure leak both. + fn drop(&mut self) { + if unsafe { (self.shared.api.file_free)(self.raw.as_ptr()) } != 0 { + self.shared.leak(); + std::mem::forget(self.fd.take()); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + /// Without a backend in scope or built in, loading fails cleanly. + fn load_without_backend_is_unavailable() { + #[cfg(not(feature = "native"))] + assert_eq!(Gpu::load().err(), Some(Error::Unavailable)); + } + + #[test] + /// Compute mode and capabilities gate usability. + fn usable_requires_default_mode_and_vmm_fd_support() { + let mut device = DeviceInfo { + ordinal: 0, + uuid: [0xab; 16], + pci_bus_id: "0000:3b:00.0".into(), + numa_node: Some(1), + compute_mode: ComputeMode::Default, + flags: flag::VMM | flag::POSIX_FD, + total_memory: 1, + }; + assert!(device.usable()); + assert_eq!( + device.uuid_string(), + "GPU-abababab-abab-abab-abab-abababababab" + ); + for mode in [ + ComputeMode::Prohibited, + ComputeMode::ExclusiveProcess, + ComputeMode::Other(-1), + ] { + device.compute_mode = mode; + assert!(!device.usable()); + } + device.compute_mode = ComputeMode::Default; + device.flags = flag::VMM; + assert!(!device.usable()); + } +} diff --git a/cmd/racer-dataplane/gpu/src/simulation.rs b/cmd/racer-dataplane/gpu/src/simulation.rs new file mode 100644 index 000000000..16b671753 --- /dev/null +++ b/cmd/racer-dataplane/gpu/src/simulation.rs @@ -0,0 +1,878 @@ +//! Fake GPUs for tests. No CUDA, driver or hardware needed. +//! +//! It implements the same C functions as `native/gpu_cuda.c`, so the real +//! wrappers in `lib.rs` run on top of it unchanged. Device memory is host +//! memory; a client allocation is a memfd made with +//! [`Simulation::allocate`] and passed as the shareable handle. Decryption +//! runs the serial host emulation of the kernels (`native/gcm_host.c`). +//! +//! Work runs when queued, not on synchronize, so tests check ordering with +//! [`Simulation::trace`] instead of timing. +//! +//! # Basics +//! +//! 1. Create a [`Simulation`] with [`SimDevice`]s and [`Simulation::enter`] it. +//! 2. [`crate::Gpu::load`] now uses it. +//! 3. Script one-shot failures with [`Simulation::fault`]. +//! 4. Check live resources with [`Simulation::live`]. +use crate::{ + ffi::{self, Api, RawDecrypt, RawDevice, RawVerdict, code}, + gcm, +}; + +use std::{ + cell::RefCell, + collections::{BTreeMap, VecDeque}, + ffi::{c_int, c_void}, + fs::File, + os::{ + fd::{FromRawFd, OwnedFd}, + unix::fs::{FileExt, MetadataExt}, + }, + rc::Rc, +}; + +/// Granularity imports must be a multiple of. Real GPUs use 2 MiB; smaller +/// here keeps tests light. +pub const GRANULARITY: u64 = 64 << 10; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +/// One simulated GPU. +#[derive(Clone, Debug)] +pub struct SimDevice { + /// GPU UUID. + pub uuid: [u8; 16], + + /// Lowercase PCI address. + pub pci_bus_id: String, + + /// NUMA node, if known. + pub numa_node: Option, + + /// Raw `CUcomputemode`. + pub compute_mode: i32, + + /// `flag::*` bits. + pub flags: u32, + + /// Device memory. + pub total_memory: u64, +} + +impl SimDevice { + /// A usable GPU with every capability. + pub fn new(index: u8) -> Self { + Self { + uuid: [index; 16], + pci_bus_id: format!("0000:{:02x}:00.0", 0x10 + index), + numa_node: Some(index as u32 % 2), + compute_mode: 0, + flags: ffi::flag::VMM | ffi::flag::POSIX_FD | ffi::flag::DMA_BUF | ffi::flag::GDR, + total_memory: 80 << 30, + } + } +} + +/// A shim function. Used in traces and to target faults. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Operation { + Init, + Discover, + Context, + ContextFree, + Stream, + StreamSync, + StreamFree, + Import, + ImportFree, + Staging, + StagingFree, + CopyH2d, + CopyD2d, + CopyD2h, + Decrypt, + Verdict, + FileOpen, + File, + FileFree, + StagingRegister, + StagingUnregister, + FileRead, + Dmabuf, +} + +/// One call in the trace. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Event { + /// The function. + pub operation: Operation, + + /// The handle it acted on or created, 0 if none. + pub handle: usize, + + /// Its result code. + pub result: i64, +} + +/// Count of live resources of each kind. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct Live { + pub contexts: usize, + pub streams: usize, + pub imports: usize, + pub staging: usize, + pub files: usize, +} + +/// A simulated set of GPUs, shared by clones. Deterministic. +#[derive(Clone)] +pub struct Simulation { + world: Rc>, +} + +/// Scope guard from [`Simulation::enter`]. Restores the previous one on drop. +pub struct Environment { + previous: Option, +} + +impl Drop for Environment { + /// Put back the previous simulation. + fn drop(&mut self) { + CURRENT.with(|current| *current.borrow_mut() = self.previous.take()); + } +} + +/// A retained context. +struct SimContext { + ordinal: i32, +} + +/// A stream and its last verdict. +struct SimStream { + context: usize, + + /// 0 none, 1 queued, 2 synchronized, as in the shim. + decrypt: u8, + + verdict: RawVerdict, +} + +/// Device memory: client memfd or staging bytes. +enum Memory { + Import { file: File }, + Staging { bytes: Vec, registered: bool }, +} + +/// One buffer. +struct Buffer { + context: usize, + + size: u64, + + memory: Memory, +} + +/// Everything the simulation tracks. +#[derive(Default)] +struct World { + devices: Vec, + + next: usize, + + contexts: BTreeMap, + + streams: BTreeMap, + + buffers: BTreeMap, + + files: BTreeMap, + + /// memfd inode to device ordinal. + allocations: BTreeMap<(u64, u64), i32>, + + faults: VecDeque<(Operation, c_int)>, + + trace: Vec, + + initialized: bool, + + file_driver: bool, + + no_kernels: bool, + + last_error: i32, +} + +impl Simulation { + /// GPUs that discovery will report, in ordinal order. + pub fn new(devices: Vec) -> Self { + Self { + world: Rc::new(RefCell::new(World { + devices, + next: 0x1000, + ..World::default() + })), + } + } + + /// Make this the simulation [`crate::Gpu::load`] uses on this thread. + pub fn enter(&self) -> Environment { + let previous = CURRENT.with(|current| current.borrow_mut().replace(self.clone())); + Environment { previous } + } + + /// A client allocation of `size` bytes on GPU `ordinal`, as the fd + /// `cuMemExportToShareableHandle` would return. Keep a clone to read it. + pub fn allocate(&self, ordinal: i32, size: u64) -> OwnedFd { + let fd = unsafe { libc::memfd_create(c"racer-gpu-sim".as_ptr(), libc::MFD_CLOEXEC) }; + assert!(fd >= 0, "memfd_create failed"); + let file = unsafe { File::from_raw_fd(fd) }; + file.set_len(size).expect("size memfd"); + let meta = file.metadata().expect("stat memfd"); + self.world + .borrow_mut() + .allocations + .insert((meta.dev(), meta.ino()), ordinal); + file.into() + } + + /// Fail the next call to `operation` with `code` (a `gpu_cuda.h` code; + /// `-5` is a driver error). Nothing changes. + pub fn fault(&self, operation: Operation, code: c_int) { + self.world.borrow_mut().faults.push_back((operation, code)); + } + + /// Behave like a shim built without decrypt kernels. + pub fn disable_kernels(&self) { + self.world.borrow_mut().no_kernels = true; + } + + /// Every call so far. + pub fn trace(&self) -> Vec { + self.world.borrow().trace.clone() + } + + /// Live resources. + pub fn live(&self) -> Live { + let world = self.world.borrow(); + let (mut imports, mut staging) = (0, 0); + for buffer in world.buffers.values() { + match buffer.memory { + Memory::Import { .. } => imports += 1, + Memory::Staging { .. } => staging += 1, + } + } + Live { + contexts: world.contexts.len(), + streams: world.streams.len(), + imports, + staging, + files: world.files.len(), + } + } +} + +/// The simulation in scope on this thread. +pub(crate) fn current() -> Option { + CURRENT.with(|current| current.borrow().clone()) +} + +/// Run `f` on the current world, recording the call and applying a fault. +fn with + Copy + From>( + operation: Operation, + f: impl FnOnce(&mut World) -> (R, usize), +) -> R { + let Some(simulation) = current() else { + return R::from(code::EUNAVAILABLE); + }; + let mut world = simulation.world.borrow_mut(); + let fault = world + .faults + .iter() + .position(|(op, _)| *op == operation) + .and_then(|i| world.faults.remove(i)); + let (result, handle) = match fault { + Some((_, rc)) => { + if rc == code::EDRIVER { + world.last_error = 999; + } + (R::from(rc), 0) + } + None => f(&mut world), + }; + world.trace.push(Event { + operation, + handle, + result: result.into(), + }); + result +} + +/// Store a new handle. +fn give(world: &mut World, out: ffi::Out) -> usize { + world.next += 0x10; + unsafe { *out = std::ptr::without_provenance_mut(world.next) }; + world.next +} + +/// Handle value of an opaque pointer. +fn id(pointer: *mut c_void) -> usize { + pointer.addr() +} + +/// Nonempty `[offset, offset + length)` inside `size`. +fn span(size: u64, offset: u64, length: u64) -> bool { + length > 0 && offset <= size && length <= size - offset +} + +unsafe extern "C" fn last_driver_error() -> i32 { + current().map_or(0, |s| s.world.borrow().last_error) +} + +unsafe extern "C" fn init() -> c_int { + with(Operation::Init, |w| { + w.initialized = true; + (code::OK, 0) + }) +} + +unsafe extern "C" fn discover(out: *mut RawDevice, capacity: u32) -> c_int { + with(Operation::Discover, |w| { + if !w.initialized { + return (code::EUNAVAILABLE, 0); + } + for (i, device) in w.devices.iter().enumerate().take(capacity as usize) { + let mut raw = RawDevice { + uuid: device.uuid, + ordinal: i as i32, + numa_node: device.numa_node.map_or(-1, |n| n as i32), + compute_mode: device.compute_mode, + flags: device.flags, + total_memory: device.total_memory, + ..RawDevice::default() + }; + let bytes = device.pci_bus_id.as_bytes(); + raw.pci_bus_id[..bytes.len().min(31)].copy_from_slice(&bytes[..bytes.len().min(31)]); + unsafe { *out.add(i) = raw }; + } + (w.devices.len() as c_int, 0) + }) +} + +unsafe extern "C" fn context(ordinal: i32, out: ffi::Out) -> c_int { + with(Operation::Context, |w| { + if ordinal < 0 || ordinal as usize >= w.devices.len() { + w.last_error = 101; // CUDA_ERROR_INVALID_DEVICE + return (code::EDRIVER, 0); + } + let handle = give(w, out); + w.contexts.insert(handle, SimContext { ordinal }); + (code::OK, handle) + }) +} + +unsafe extern "C" fn context_free(context: *mut c_void) -> c_int { + with(Operation::ContextFree, |w| { + let handle = id(context); + let used = w.streams.values().any(|s| s.context == handle) + || w.buffers.values().any(|b| b.context == handle); + if used { + // Rust keeps the context alive while anything uses it; refuse + // so a bug shows up as a counted leak. + return (code::EBUSY, handle); + } + match w.contexts.remove(&handle) { + Some(_) => (code::OK, handle), + None => (code::EINVAL, handle), + } + }) +} + +unsafe extern "C" fn stream(context: *mut c_void, out: ffi::Out) -> c_int { + with(Operation::Stream, |w| { + if !w.contexts.contains_key(&id(context)) { + return (code::EINVAL, 0); + } + let handle = give(w, out); + w.streams.insert( + handle, + SimStream { + context: id(context), + decrypt: 0, + verdict: RawVerdict::default(), + }, + ); + (code::OK, handle) + }) +} + +unsafe extern "C" fn stream_sync(stream: *mut c_void) -> c_int { + with(Operation::StreamSync, |w| { + match w.streams.get_mut(&id(stream)) { + Some(s) => { + if s.decrypt == 1 { + s.decrypt = 2; + } + (code::OK, id(stream)) + } + None => (code::EINVAL, id(stream)), + } + }) +} + +unsafe extern "C" fn stream_free(stream: *mut c_void) -> c_int { + with(Operation::StreamFree, |w| { + match w.streams.remove(&id(stream)) { + Some(_) => (code::OK, id(stream)), + None => (code::EINVAL, id(stream)), + } + }) +} + +unsafe extern "C" fn import(context: *mut c_void, fd: c_int, size: u64, out: ffi::Out) -> c_int { + with(Operation::Import, |w| { + let Some(ctx) = w.contexts.get(&id(context)) else { + return (code::EINVAL, 0); + }; + let ordinal = ctx.ordinal; + if fd < 0 || size == 0 { + return (code::EINVAL, 0); + } + // Like cuMemImportFromShareableHandle, keep our own reference. + let duplicate = unsafe { libc::fcntl(fd, libc::F_DUPFD_CLOEXEC, 0) }; + if duplicate < 0 { + w.last_error = 1; // CUDA_ERROR_INVALID_VALUE + return (code::EDRIVER, 0); + } + let file = unsafe { File::from_raw_fd(duplicate) }; + let Ok(meta) = file.metadata() else { + return (code::EDRIVER, 0); + }; + let Some(&owner) = w.allocations.get(&(meta.dev(), meta.ino())) else { + w.last_error = 1; + return (code::EDRIVER, 0); + }; + if owner != ordinal { + return (code::EDEVICE, 0); + } + if !size.is_multiple_of(GRANULARITY) { + return (code::EINVAL, 0); + } + if size > meta.len() { + w.last_error = 1; + return (code::EDRIVER, 0); + } + let handle = give(w, out); + w.buffers.insert( + handle, + Buffer { + context: id(context), + size, + memory: Memory::Import { file }, + }, + ); + (code::OK, handle) + }) +} + +unsafe extern "C" fn import_free(allocation: *mut c_void) -> c_int { + with(Operation::ImportFree, |w| { + let handle = id(allocation); + match w.buffers.get(&handle) { + Some(Buffer { + memory: Memory::Import { .. }, + .. + }) => { + w.buffers.remove(&handle); + (code::OK, handle) + } + _ => (code::EINVAL, handle), + } + }) +} + +unsafe extern "C" fn staging(context: *mut c_void, size: u64, out: ffi::Out) -> c_int { + with(Operation::Staging, |w| { + if !w.contexts.contains_key(&id(context)) || size == 0 { + return (code::EINVAL, 0); + } + if size > 1 << 32 { + return (code::ENOMEM, 0); + } + let handle = give(w, out); + w.buffers.insert( + handle, + Buffer { + context: id(context), + size, + memory: Memory::Staging { + bytes: vec![0; size as usize], + registered: false, + }, + }, + ); + (code::OK, handle) + }) +} + +unsafe extern "C" fn staging_free(staging: *mut c_void) -> c_int { + with(Operation::StagingFree, |w| { + let handle = id(staging); + match w.buffers.get(&handle) { + Some(Buffer { + memory: Memory::Staging { registered, .. }, + .. + }) => { + if *registered { + return (code::EBUSY, handle); + } + w.buffers.remove(&handle); + (code::OK, handle) + } + _ => (code::EINVAL, handle), + } + }) +} + +impl World { + /// Check that a stream and buffer exist and share a context. + fn pair(&self, stream: *mut c_void, buffer: *mut c_void) -> Result<(), c_int> { + let s = self.streams.get(&id(stream)).ok_or(code::EINVAL)?; + let b = self.buffers.get(&id(buffer)).ok_or(code::EINVAL)?; + if s.context != b.context { + return Err(code::EDEVICE); + } + Ok(()) + } + + /// Write bytes into a buffer, bounds already checked. + fn write(&mut self, buffer: usize, offset: u64, bytes: &[u8]) -> c_int { + match &mut self.buffers.get_mut(&buffer).expect("checked").memory { + Memory::Import { file } => match file.write_all_at(bytes, offset) { + Ok(()) => code::OK, + Err(_) => code::EDRIVER, + }, + Memory::Staging { bytes: memory, .. } => { + memory[offset as usize..offset as usize + bytes.len()].copy_from_slice(bytes); + code::OK + } + } + } + + /// Staging bytes, or None for an import. + fn staging(&self, buffer: usize) -> Option<&[u8]> { + match &self.buffers.get(&buffer)?.memory { + Memory::Staging { bytes, .. } => Some(bytes), + Memory::Import { .. } => None, + } + } +} + +unsafe extern "C" fn copy_h2d( + stream: *mut c_void, + dst: *mut c_void, + offset: u64, + src: *const c_void, + length: u64, +) -> c_int { + with(Operation::CopyH2d, |w| { + if let Err(rc) = w.pair(stream, dst) { + return (rc, id(dst)); + } + if src.is_null() || !span(w.buffers[&id(dst)].size, offset, length) { + return (code::ERANGE, id(dst)); + } + let bytes = unsafe { std::slice::from_raw_parts(src.cast::(), length as usize) }; + (w.write(id(dst), offset, bytes), id(dst)) + }) +} + +unsafe extern "C" fn copy_d2d( + stream: *mut c_void, + dst: *mut c_void, + dst_offset: u64, + src: *mut c_void, + src_offset: u64, + length: u64, +) -> c_int { + with(Operation::CopyD2d, |w| { + if let Err(rc) = w.pair(stream, dst).and(w.pair(stream, src)) { + return (rc, id(dst)); + } + let Some(from) = w.staging(id(src)) else { + return (code::EINVAL, id(dst)); + }; + if !span(w.buffers[&id(dst)].size, dst_offset, length) + || !span(from.len() as u64, src_offset, length) + { + return (code::ERANGE, id(dst)); + } + let bytes = from[src_offset as usize..(src_offset + length) as usize].to_vec(); + (w.write(id(dst), dst_offset, &bytes), id(dst)) + }) +} + +unsafe extern "C" fn copy_d2h( + stream: *mut c_void, + dst: *mut c_void, + src: *mut c_void, + src_offset: u64, + length: u64, +) -> c_int { + with(Operation::CopyD2h, |w| { + if let Err(rc) = w.pair(stream, src) { + return (rc, id(src)); + } + let Some(from) = w.staging(id(src)) else { + return (code::EINVAL, id(src)); + }; + if dst.is_null() || !span(from.len() as u64, src_offset, length) { + return (code::ERANGE, id(src)); + } + let out = unsafe { std::slice::from_raw_parts_mut(dst.cast::(), length as usize) }; + out.copy_from_slice(&from[src_offset as usize..(src_offset + length) as usize]); + let s = w.streams.get_mut(&id(stream)).expect("checked"); + if s.decrypt == 1 { + s.decrypt = 2; + } + (code::OK, id(src)) + }) +} + +unsafe extern "C" fn decrypt( + stream: *mut c_void, + params: *const RawDecrypt, + src: *mut c_void, + src_offset: u64, + dst: *mut c_void, + dst_offset: u64, +) -> c_int { + with(Operation::Decrypt, |w| { + if params.is_null() { + return (code::EINVAL, 0); + } + let p = unsafe { &*params }; + if let Err(rc) = w.pair(stream, dst).and(w.pair(stream, src)) { + return (rc, id(dst)); + } + if p.length > ffi::MAX_DECRYPT + || p.aad_length > ffi::MAX_AAD as u64 + || (p.aad.is_null() && p.aad_length > 0) + || p.check_crc > 1 + { + return (code::EINVAL, id(dst)); + } + let Some(from) = w.staging(id(src)) else { + return (code::EINVAL, id(dst)); + }; + let n = p.length; + let dst_size = w.buffers[&id(dst)].size; + if !span(from.len() as u64, src_offset, n + 16) + || (n > 0 && !span(dst_size, dst_offset, n)) + || (n == 0 && dst_offset > dst_size) + { + return (code::ERANGE, id(dst)); + } + if w.streams[&id(stream)].decrypt == 1 { + return (code::EBUSY, id(dst)); + } + if w.no_kernels { + return (code::ENOKERNEL, id(dst)); + } + let sealed = &from[src_offset as usize..(src_offset + n + 16) as usize]; + let aad = if p.aad_length == 0 { + &[][..] + } else { + unsafe { std::slice::from_raw_parts(p.aad, p.aad_length as usize) } + }; + let mut out = zeroize::Zeroizing::new(vec![0; n as usize]); + let expected = (p.check_crc == 1).then_some(p.expected_crc); + let Some(verdict) = gcm::decrypt(&p.key, &p.nonce, aad, sealed, expected, &mut out) else { + return (code::ENOMEM, id(dst)); + }; + let mut rc = code::OK; + if verdict.status == gcm::VERDICT_PASS && n > 0 { + rc = w.write(id(dst), dst_offset, &out); + } + let s = w.streams.get_mut(&id(stream)).expect("checked"); + s.decrypt = 1; + s.verdict = verdict; + (rc, id(dst)) + }) +} + +unsafe extern "C" fn verdict(stream: *mut c_void, out: *mut RawVerdict) -> c_int { + with(Operation::Verdict, |w| { + let Some(s) = w.streams.get_mut(&id(stream)) else { + return (code::EINVAL, 0); + }; + if s.decrypt != 2 || out.is_null() { + return (code::EBUSY, id(stream)); + } + unsafe { *out = std::mem::take(&mut s.verdict) }; + s.decrypt = 0; + (code::OK, id(stream)) + }) +} + +unsafe extern "C" fn file_open() -> c_int { + with(Operation::FileOpen, |w| { + w.file_driver = true; + (code::OK, 0) + }) +} + +unsafe extern "C" fn file(fd: c_int, out: ffi::Out) -> c_int { + with(Operation::File, |w| { + if !w.file_driver { + return (code::EUNAVAILABLE, 0); + } + let duplicate = unsafe { libc::fcntl(fd, libc::F_DUPFD_CLOEXEC, 0) }; + if fd < 0 || duplicate < 0 { + return (code::EINVAL, 0); + } + let handle = give(w, out); + w.files + .insert(handle, unsafe { File::from_raw_fd(duplicate) }); + (code::OK, handle) + }) +} + +unsafe extern "C" fn file_free(file: *mut c_void) -> c_int { + with(Operation::FileFree, |w| match w.files.remove(&id(file)) { + Some(_) => (code::OK, id(file)), + None => (code::EINVAL, id(file)), + }) +} + +/// Set a staging buffer's cuFile registration. +fn register(w: &mut World, staging: *mut c_void, value: bool) -> (c_int, usize) { + if !w.file_driver { + return (code::EUNAVAILABLE, id(staging)); + } + match w.buffers.get_mut(&id(staging)).map(|b| &mut b.memory) { + Some(Memory::Staging { registered, .. }) if *registered != value => { + *registered = value; + (code::OK, id(staging)) + } + Some(Memory::Staging { .. }) => (code::EBUSY, id(staging)), + _ => (code::EINVAL, id(staging)), + } +} + +unsafe extern "C" fn staging_register(staging: *mut c_void) -> c_int { + with(Operation::StagingRegister, |w| register(w, staging, true)) +} + +unsafe extern "C" fn staging_unregister(staging: *mut c_void) -> c_int { + with(Operation::StagingUnregister, |w| { + register(w, staging, false) + }) +} + +unsafe extern "C" fn file_read( + file: *mut c_void, + staging: *mut c_void, + offset: u64, + file_offset: u64, + length: u64, +) -> i64 { + with(Operation::FileRead, |w| { + let Some(source) = w.files.get(&id(file)) else { + return (code::EINVAL as i64, 0); + }; + let Some(Buffer { + memory: Memory::Staging { bytes, .. }, + size, + .. + }) = w.buffers.get_mut(&id(staging)) + else { + return (code::EINVAL as i64, 0); + }; + if !span(*size, offset, length) { + return (code::ERANGE as i64, id(staging)); + } + let out = &mut bytes[offset as usize..(offset + length) as usize]; + match source.read_exact_at(out, file_offset) { + Ok(()) => (length as i64, id(staging)), + Err(_) => (code::EIO as i64, id(staging)), + } + }) +} + +unsafe extern "C" fn dmabuf( + staging: *mut c_void, + offset: u64, + length: u64, + fd: *mut c_int, +) -> c_int { + with(Operation::Dmabuf, |w| { + let Some(size) = w.staging(id(staging)).map(|b| b.len() as u64) else { + return (code::EINVAL, id(staging)); + }; + if fd.is_null() { + return (code::EINVAL, id(staging)); + } + let page = 4096; + if length == 0 + || !offset.is_multiple_of(page) + || !length.is_multiple_of(page) + || offset + length > size + { + return (code::ERANGE, id(staging)); + } + // Not a live view of the buffer: a placeholder fd of the right size. + let memfd = unsafe { libc::memfd_create(c"racer-gpu-dmabuf".as_ptr(), libc::MFD_CLOEXEC) }; + if memfd < 0 { + return (code::EDRIVER, id(staging)); + } + let file = unsafe { File::from_raw_fd(memfd) }; + if file.set_len(length).is_err() { + return (code::EDRIVER, id(staging)); + } + unsafe { *fd = std::os::fd::IntoRawFd::into_raw_fd(file) }; + (code::OK, id(staging)) + }) +} + +/// The simulation's function table. +pub(crate) fn api() -> Rc { + Rc::new(Api { + library: None, + last_driver_error, + init, + discover, + context, + context_free, + stream, + stream_sync, + stream_free, + import, + import_free, + staging, + staging_free, + copy_h2d, + copy_d2d, + copy_d2h, + decrypt, + verdict, + file_open, + file, + file_free, + staging_register, + staging_unregister, + file_read, + dmabuf, + }) +} + +/// Read `length` bytes at `offset` of a client allocation fd. +pub fn read_allocation(fd: &OwnedFd, offset: u64, length: usize) -> Vec { + let file = File::from(fd.try_clone().expect("dup")); + let mut out = vec![0; length]; + file.read_exact_at(&mut out, offset).expect("read memfd"); + out +} diff --git a/cmd/racer-dataplane/gpu/tests/simulation.rs b/cmd/racer-dataplane/gpu/tests/simulation.rs new file mode 100644 index 000000000..147e2eb25 --- /dev/null +++ b/cmd/racer-dataplane/gpu/tests/simulation.rs @@ -0,0 +1,445 @@ +//! Safe API behavior on the simulation backend: discovery, import windows, +//! decrypt verdicts, lifetimes, fault injection and leak counting. +#![cfg(feature = "simulation")] + +use racer_gpu::{ + DecryptRequest, Destination, Error, Gpu, Verdict, + simulation::{GRANULARITY, Operation, SimDevice, Simulation, read_allocation}, +}; +use std::os::fd::OwnedFd; + +/// Two usable GPUs and one in EXCLUSIVE_PROCESS mode. +fn simulation() -> Simulation { + let mut exclusive = SimDevice::new(2); + exclusive.compute_mode = 3; + Simulation::new(vec![SimDevice::new(0), SimDevice::new(1), exclusive]) +} + +/// A memfd with arbitrary bytes, standing in for an O_DIRECT file. +fn file_with(bytes: &[u8]) -> OwnedFd { + use std::os::unix::fs::FileExt; + let fd = unsafe { libc::memfd_create(c"file".as_ptr(), libc::MFD_CLOEXEC) }; + assert!(fd >= 0); + let file = unsafe { ::from_raw_fd(fd) }; + file.write_all_at(bytes, 0).unwrap(); + file.into() +} + +/// A sealed page and what the GPU needs to open it. +struct Page { + derived: racer_crypto::DerivedPageKey, + aad: Vec, + plaintext: Vec, + sealed: Vec, +} + +/// Seal `length` bytes with XAES-256-GCM. +fn page(length: usize) -> Page { + let key = [9; 32]; + let nonce = *b"0123456789abcdefghijklmn"; + let aad = b"racer/page/aead/v1\0test".to_vec(); + let plaintext: Vec = (0..length).map(|i| (i * 7 + 3) as u8).collect(); + let mut sealed = vec![0; length + 16]; + racer_crypto::seal(&key, &nonce, &aad, &plaintext, &mut sealed).unwrap(); + Page { + derived: racer_crypto::xaes_derive(&key, &nonce), + aad, + plaintext, + sealed, + } +} + +impl Page { + /// The GPU request. + fn request(&self, expected_crc: Option) -> DecryptRequest<'_> { + DecryptRequest { + key: self.derived.key(), + nonce: self.derived.nonce(), + aad: &self.aad, + length: self.plaintext.len() as u64, + expected_crc, + } + } +} + +#[test] +/// Discovery reports every GPU; only default-mode GPUs are usable. +fn discovery_reports_all_and_filters_unusable() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let devices = gpu.devices().unwrap(); + assert_eq!(devices.len(), 3); + assert_eq!(devices[1].pci_bus_id, "0000:11:00.0"); + assert_eq!(devices[1].numa_node, Some(1)); + let usable: Vec<_> = gpu + .usable_devices() + .unwrap() + .iter() + .map(|d| d.ordinal) + .collect(); + assert_eq!(usable, [0, 1]); +} + +#[test] +/// Init and context failures surface the driver status. +fn faults_surface_driver_errors() { + let sim = simulation(); + let _env = sim.enter(); + sim.fault(Operation::Init, -3); + assert_eq!(Gpu::load().err(), Some(Error::Unavailable)); + let gpu = Gpu::load().unwrap(); + sim.fault(Operation::Context, -5); + assert_eq!(gpu.context(0).err(), Some(Error::Driver(999))); + assert_eq!(gpu.context(7).err(), Some(Error::Driver(101))); + assert_eq!(sim.live().contexts, 0); +} + +#[test] +/// Writes are limited to the request window and the import's GPU. +fn import_rejects_out_of_window_and_wrong_device() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let other = gpu.context(1).unwrap(); + let size = 2 * GRANULARITY; + let client = sim.allocate(0, size); + + // Bad windows, granularity, and devices fail before or at import. + let dup = || client.try_clone().unwrap(); + assert_eq!( + ctx.import(dup(), size, 10..10).err(), + Some(Error::InvalidRange) + ); + assert_eq!( + ctx.import(dup(), size, 0..size + 1).err(), + Some(Error::InvalidRange) + ); + assert_eq!( + ctx.import(dup(), size - 1, 0..1).err(), + Some(Error::InvalidRequest) + ); + assert_eq!( + ctx.import(dup(), 4 * GRANULARITY, 0..1).err(), + Some(Error::Driver(1)) + ); + assert_eq!( + other.import(dup(), size, 0..1).err(), + Some(Error::WrongDevice) + ); + let stranger = file_with(&[0; 16]); + assert_eq!( + ctx.import(stranger, size, 0..1).err(), + Some(Error::Driver(1)) + ); + + let import = ctx.import(dup(), size, 100..200).unwrap(); + assert_eq!(import.window(), 100..200); + let stream = ctx.stream().unwrap(); + let foreign = other.stream().unwrap(); + let dst = Destination::Import(&import); + stream.copy_from_host(dst, 100, &[7; 100]).unwrap(); + assert_eq!( + stream.copy_from_host(dst, 99, &[1; 2]), + Err(Error::InvalidRange) + ); + assert_eq!( + stream.copy_from_host(dst, 150, &[1; 51]), + Err(Error::InvalidRange) + ); + assert_eq!( + stream.copy_from_host(dst, u64::MAX, &[1]), + Err(Error::InvalidRange) + ); + assert_eq!( + stream.copy_from_host(dst, 100, &[]), + Err(Error::InvalidRange) + ); + assert_eq!( + foreign.copy_from_host(dst, 100, &[1]), + Err(Error::WrongDevice) + ); + stream.synchronize().unwrap(); + let bytes = read_allocation(&client, 98, 104); + assert_eq!(&bytes[..2], &[0, 0]); + assert!(bytes[2..102].iter().all(|&b| b == 7)); + assert_eq!(&bytes[102..], &[0, 0]); +} + +#[test] +/// Ciphertext staged on the GPU decrypts into the client window; tampered +/// pages and wrong CRCs leave it untouched. +fn decrypt_into_import_verifies_before_writing() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(1).unwrap(); + let client = sim.allocate(1, GRANULARITY); + let import = ctx + .import(client.try_clone().unwrap(), GRANULARITY, 64..64 + 5000) + .unwrap(); + let staging = ctx.staging(8192).unwrap(); + let stream = ctx.stream().unwrap(); + let page = page(5000); + let crc = racer_crypto::crc64(&page.sealed); + let dst = Destination::Import(&import); + + assert_eq!(stream.verdict(), Err(Error::Busy)); + stream + .copy_from_host(Destination::Staging(&staging), 16, &page.sealed) + .unwrap(); + stream + .decrypt(&page.request(Some(crc)), &staging, 16, dst, 64) + .unwrap(); + // One decrypt per synchronize. + assert_eq!( + stream.decrypt(&page.request(None), &staging, 16, dst, 64), + Err(Error::Busy) + ); + stream.synchronize().unwrap(); + assert_eq!(stream.verdict(), Ok(Verdict::Passed { crc })); + assert_eq!(stream.verdict(), Err(Error::Busy)); + assert_eq!(read_allocation(&client, 64, 5000), page.plaintext); + + // Fresh destination for the failure cases. + let client2 = sim.allocate(1, GRANULARITY); + let import2 = ctx + .import(client2.try_clone().unwrap(), GRANULARITY, 0..5000) + .unwrap(); + let dst2 = Destination::Import(&import2); + stream + .decrypt(&page.request(Some(crc ^ 1)), &staging, 16, dst2, 0) + .unwrap(); + stream.synchronize().unwrap(); + assert_eq!(stream.verdict(), Ok(Verdict::CrcMismatch { crc })); + + let mut tampered = page.sealed.clone(); + tampered[4999] ^= 1; + stream + .copy_from_host(Destination::Staging(&staging), 16, &tampered) + .unwrap(); + stream + .decrypt(&page.request(None), &staging, 16, dst2, 0) + .unwrap(); + stream.synchronize().unwrap(); + assert!(matches!(stream.verdict(), Ok(Verdict::TagMismatch { .. }))); + assert!(read_allocation(&client2, 0, 5000).iter().all(|&b| b == 0)); + + // Out-of-window destination and out-of-range source are refused. + assert_eq!( + stream.decrypt(&page.request(None), &staging, 16, dst2, 1), + Err(Error::InvalidRange) + ); + assert_eq!( + stream.decrypt(&page.request(None), &staging, 8192 - 5000, dst2, 0), + Err(Error::InvalidRange) + ); + let mut big = page.request(None); + big.length = racer_gpu::MAX_DECRYPT + 1; + assert_eq!( + stream.decrypt(&big, &staging, 0, dst2, 0), + Err(Error::InvalidRequest) + ); + let long_aad = vec![0; racer_gpu::MAX_AAD + 1]; + let mut long = page.request(None); + long.aad = &long_aad; + assert_eq!( + stream.decrypt(&long, &staging, 0, dst2, 0), + Err(Error::InvalidRequest) + ); +} + +#[test] +/// A shim without kernels reports it instead of decrypting. +fn decrypt_without_kernels_is_unavailable() { + let sim = simulation(); + sim.disable_kernels(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let staging = ctx.staging(64).unwrap(); + let out = ctx.staging(64).unwrap(); + let stream = ctx.stream().unwrap(); + let page = page(16); + assert_eq!( + stream.decrypt( + &page.request(None), + &staging, + 0, + Destination::Staging(&out), + 0 + ), + Err(Error::KernelUnavailable) + ); +} + +#[test] +/// Dropping an import while work is queued defers the unmap to after the +/// stream synchronizes. +fn import_is_released_only_after_stream_sync() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let client = sim.allocate(0, GRANULARITY); + let import = ctx.import(client, GRANULARITY, 0..10).unwrap(); + let stream = ctx.stream().unwrap(); + stream + .copy_from_host(Destination::Import(&import), 0, b"0123456789") + .unwrap(); + drop(import); + assert_eq!(sim.live().imports, 1); + stream.synchronize().unwrap(); + assert_eq!(sim.live().imports, 0); + let order: Vec<_> = sim + .trace() + .iter() + .map(|e| e.operation) + .filter(|op| matches!(op, Operation::StreamSync | Operation::ImportFree)) + .collect(); + assert_eq!(order, [Operation::StreamSync, Operation::ImportFree]); + + // Dropping the stream itself synchronizes before releasing holds. + let client = sim.allocate(0, GRANULARITY); + let import = ctx.import(client, GRANULARITY, 0..10).unwrap(); + stream + .copy_from_host(Destination::Import(&import), 0, b"x") + .unwrap(); + drop(import); + drop(stream); + assert_eq!(sim.live().imports, 0); + drop(ctx); + assert_eq!(sim.live().contexts, 0); + assert_eq!(gpu.leaked(), 0); +} + +#[test] +/// Failed frees leak the resource and count it; nothing is freed early. +fn failed_frees_leak_and_count() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let import = ctx + .import(sim.allocate(0, GRANULARITY), GRANULARITY, 0..1) + .unwrap(); + sim.fault(Operation::ImportFree, -5); + drop(import); + assert_eq!(gpu.leaked(), 1); + assert_eq!(sim.live().imports, 1); + + // A stream that cannot be freed keeps everything it touched. + let stream = ctx.stream().unwrap(); + let staging = ctx.staging(16).unwrap(); + stream + .copy_from_host(Destination::Staging(&staging), 0, &[1; 16]) + .unwrap(); + assert!(staging.in_flight()); + sim.fault(Operation::StreamFree, -5); + drop(stream); + drop(staging); + assert_eq!(gpu.leaked(), 2); + assert_eq!(sim.live().streams, 1); + assert_eq!(sim.live().staging, 1); + + // A failed synchronize keeps holds; the buffer cannot be freed yet. + let stream = ctx.stream().unwrap(); + let staging = ctx.staging(16).unwrap(); + stream + .copy_from_host(Destination::Staging(&staging), 0, &[1; 16]) + .unwrap(); + sim.fault(Operation::StreamSync, -5); + assert_eq!(stream.synchronize(), Err(Error::Driver(999))); + assert!(staging.in_flight()); + stream.synchronize().unwrap(); + assert!(!staging.in_flight()); + + // Context leaks when release fails. + let leaked_ctx = gpu.context(1).unwrap(); + sim.fault(Operation::ContextFree, -5); + drop(leaked_ctx); + assert_eq!(gpu.leaked(), 3); +} + +#[test] +/// cuFile reads land in staging, respect bounds, and wait for streams. +fn cufile_reads_into_staging() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let staging = ctx.staging(8192).unwrap(); + staging.register_cufile().unwrap(); + let data: Vec = (0..10000u32).map(|i| i as u8).collect(); + let file = gpu.open_file(file_with(&data)).unwrap(); + file.read(&staging, 100, 4096, 1000).unwrap(); + let stream = ctx.stream().unwrap(); + let mut out = vec![0; 1000]; + stream.read_staging(&staging, 100, &mut out).unwrap(); + assert_eq!(out, &data[4096..5096]); + assert_eq!(file.read(&staging, 8000, 0, 200), Err(Error::InvalidRange)); + assert_eq!(file.read(&staging, 0, 0, 0), Err(Error::InvalidRange)); + assert_eq!(file.read(&staging, 0, 9990, 100), Err(Error::Io)); + + // A queued copy holds the buffer. + stream + .copy_from_host(Destination::Staging(&staging), 0, &[1]) + .unwrap(); + assert_eq!(file.read(&staging, 0, 0, 1), Err(Error::Busy)); + stream.synchronize().unwrap(); + file.read(&staging, 0, 0, 1).unwrap(); + drop(file); + drop(staging); + assert_eq!(sim.live().files, 0); + assert_eq!(sim.live().staging, 0); + assert_eq!(gpu.leaked(), 0); +} + +#[test] +/// dma-buf export checks page alignment and bounds. +fn dma_buf_export_checks_alignment() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let staging = ctx.staging(16384).unwrap(); + staging.dma_buf(4096, 8192).unwrap(); + assert_eq!(staging.dma_buf(1, 4096).err(), Some(Error::InvalidRange)); + assert_eq!(staging.dma_buf(0, 100).err(), Some(Error::InvalidRange)); + assert_eq!( + staging.dma_buf(8192, 16384).err(), + Some(Error::InvalidRange) + ); +} + +#[test] +/// Device copies move staged bytes into the window; reads of staging work. +fn device_copy_from_staging() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let client = sim.allocate(0, GRANULARITY); + let import = ctx + .import(client.try_clone().unwrap(), GRANULARITY, 10..20) + .unwrap(); + let staging = ctx.staging(32).unwrap(); + let stream = ctx.stream().unwrap(); + stream + .copy_from_host(Destination::Staging(&staging), 0, b"abcdefghij") + .unwrap(); + stream + .copy_from_staging(Destination::Import(&import), 10, &staging, 0, 10) + .unwrap(); + assert_eq!( + stream.copy_from_staging(Destination::Import(&import), 10, &staging, 0, 11), + Err(Error::InvalidRange) + ); + assert_eq!( + stream.copy_from_staging(Destination::Import(&import), 10, &staging, 30, 5), + Err(Error::InvalidRange) + ); + stream.synchronize().unwrap(); + assert_eq!(read_allocation(&client, 10, 10), b"abcdefghij"); +} From c3f662a522ab5e29997dd3b471da7ec8c4a69226 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 19:22:40 +0000 Subject: [PATCH 16/87] feat(racer): add cuda feature enabling the racer-gpu native shim --- cmd/racer-dataplane/Cargo.lock | 1 + cmd/racer-dataplane/Cargo.toml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/cmd/racer-dataplane/Cargo.lock b/cmd/racer-dataplane/Cargo.lock index 603b35cdd..4f73d3ee6 100644 --- a/cmd/racer-dataplane/Cargo.lock +++ b/cmd/racer-dataplane/Cargo.lock @@ -868,6 +868,7 @@ dependencies = [ "prost", "racer-control-wire", "racer-crypto", + "racer-gpu", "racer-object-wire", "rcgen", "rdma-verbs", diff --git a/cmd/racer-dataplane/Cargo.toml b/cmd/racer-dataplane/Cargo.toml index 475cd8a79..1beab987b 100644 --- a/cmd/racer-dataplane/Cargo.toml +++ b/cmd/racer-dataplane/Cargo.toml @@ -14,6 +14,8 @@ description = "Racer node-local read-through cache dataplane" default = [] # Enables the separately loaded native verbs adapter; unavailable hardware uses HTTP. rdma = ["rdma-verbs/native"] +# Enables the separately loaded CUDA shim (libracer_gpu_cuda.so.1); without a GPU, gpu-read is unavailable. +cuda = ["racer-gpu/native"] # Test-only Go SDK launcher; keeps crate-private fixture assembly out of default builds. subscription-interop = [] @@ -36,6 +38,7 @@ telemetry = { path = "telemetry" } http1 = { path = "http" } racer-crypto = { path = "crypto" } rdma-verbs = { path = "verbs" } +racer-gpu = { path = "gpu" } page-alloc = { path = "alloc" } uring-runtime = { path = "runtime" } base64 = "0.22" @@ -61,6 +64,7 @@ uds-endpoint = { path = "uds-endpoint", features = ["test-util", "simulation"] } flow-control = { path = "flow", features = ["simulation"] } wire-codec = { path = "wire-codec", features = ["simulation"] } rdma-verbs = { path = "verbs", features = ["simulation"] } +racer-gpu = { path = "gpu", features = ["simulation"] } http1 = { path = "http", features = ["test-util"] } page-alloc = { path = "alloc", features = ["simulation"] } uring-runtime = { path = "runtime", features = ["simulation", "test-util"] } From 98d55e15488fe1e485189563e4a7df8c1193ccaa Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 19:23:01 +0000 Subject: [PATCH 17/87] build(racer): add RACER_NATIVE_CUDA image build argument --- images/racer-dataplane/Containerfile | 43 ++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/images/racer-dataplane/Containerfile b/images/racer-dataplane/Containerfile index 463fb131f..72a6a1dd4 100644 --- a/images/racer-dataplane/Containerfile +++ b/images/racer-dataplane/Containerfile @@ -7,6 +7,11 @@ ARG RUNTIME_IMAGE=ubuntu:noble # Build on the target platform: both Rust and the verbs adapter use its ABI. FROM ${RUST_IMAGE} AS builder ARG RACER_NATIVE_RDMA=true +# Build the CUDA shim (libracer_gpu_cuda.so.1) and the `cuda` feature. The +# shim dlopens the driver at runtime, so only nvcc (for the decrypt kernel PTX) +# is installed here; the runtime image gets the driver from the NVIDIA runtime. +ARG RACER_NATIVE_CUDA=false +ARG CUDA_NVCC_PACKAGE=cuda-nvcc-12-9 ENV RUSTFLAGS="-C force-frame-pointers=yes" WORKDIR /src @@ -17,6 +22,26 @@ RUN case "$RACER_NATIVE_RDMA" in \ *) echo "RACER_NATIVE_RDMA must be true or false" >&2; exit 1 ;; \ esac +RUN set -eu; \ + case "$RACER_NATIVE_CUDA" in \ + true) \ + case "$(uname -m)" in \ + x86_64) repo=x86_64 ;; \ + aarch64) repo=sbsa ;; \ + *) echo "unsupported CUDA architecture $(uname -m)" >&2; exit 1 ;; \ + esac; \ + curl -fsSL -o /tmp/cuda-keyring.deb \ + "https://developer.download.nvidia.com/compute/cuda/repos/debian12/$repo/cuda-keyring_1.1-1_all.deb"; \ + dpkg -i /tmp/cuda-keyring.deb; \ + rm /tmp/cuda-keyring.deb; \ + apt-get update; \ + apt-get install -y --no-install-recommends "$CUDA_NVCC_PACKAGE"; \ + rm -rf /var/lib/apt/lists/* ;; \ + false) ;; \ + *) echo "RACER_NATIVE_CUDA must be true or false" >&2; exit 1 ;; \ + esac +ENV PATH="${PATH}:/usr/local/cuda/bin" + # Copy only build inputs, never host target trees or deployment credentials. COPY cmd/racer-dataplane/Cargo.toml cmd/racer-dataplane/Cargo.lock ./ COPY cmd/racer-dataplane/src ./src @@ -39,6 +64,9 @@ COPY cmd/racer-dataplane/crypto/src ./crypto/src COPY cmd/racer-dataplane/verbs/Cargo.toml ./verbs/Cargo.toml COPY cmd/racer-dataplane/verbs/src ./verbs/src COPY cmd/racer-dataplane/verbs/native/verbs.c ./verbs/native/verbs.c +COPY cmd/racer-dataplane/gpu/Cargo.toml cmd/racer-dataplane/gpu/build.rs ./gpu/ +COPY cmd/racer-dataplane/gpu/src ./gpu/src +COPY cmd/racer-dataplane/gpu/native ./gpu/native COPY cmd/racer-dataplane/http/Cargo.toml ./http/Cargo.toml COPY cmd/racer-dataplane/http/src ./http/src COPY cmd/racer-dataplane/http/tests ./http/tests @@ -52,14 +80,19 @@ COPY cmd/racer-dataplane/wire-codec/Cargo.toml ./wire-codec/Cargo.toml COPY cmd/racer-dataplane/wire-codec/src ./wire-codec/src COPY cmd/racer-dataplane/wire-codec/tests ./wire-codec/tests COPY images/racer-dataplane/build-native.sh ./build-native.sh +COPY images/racer-dataplane/build-native-cuda.sh ./build-native-cuda.sh COPY images/racer-dataplane/check-debug-info.sh ./check-debug-info.sh RUN set -eu; \ mkdir -p /out/unbounded/bin /out/usr/local/lib; \ + features=; \ + if [ "$RACER_NATIVE_RDMA" = true ]; then features=rdma; fi; \ + if [ "$RACER_NATIVE_CUDA" = true ]; then features="${features:+$features,}cuda"; fi; \ + cargo build --locked --release --bin racer-dataplane --no-default-features ${features:+--features "$features"}; \ if [ "$RACER_NATIVE_RDMA" = true ]; then \ - cargo build --locked --release --bin racer-dataplane --no-default-features --features rdma; \ sh ./build-native.sh verbs/native/verbs.c /out/usr/local/lib/librdma_verbs.so.1; \ - else \ - cargo build --locked --release --bin racer-dataplane --no-default-features; \ + fi; \ + if [ "$RACER_NATIVE_CUDA" = true ]; then \ + RACER_GPU_REQUIRE_PTX=1 sh ./build-native-cuda.sh gpu/native /out/usr/local/lib/libracer_gpu_cuda.so.1; \ fi; \ sh ./check-debug-info.sh target/release/racer-dataplane; \ install -m 0755 target/release/racer-dataplane /out/unbounded/bin/racer-dataplane @@ -85,8 +118,8 @@ RUN set -eu; \ chown -R 65532:65532 /run/racer /var/lib/racer; \ chmod 0700 /var/lib/racer/identity /var/lib/racer/slabs -# Install /unbounded/bin/racer-dataplane and, with RDMA, the verbs adapter -# under /usr/local/lib where the dynamic loader finds it by soname. +# Install /unbounded/bin/racer-dataplane and, with RDMA or CUDA, the verbs +# adapter and CUDA shim under /usr/local/lib where the dynamic loader finds it by soname. COPY --from=builder /out/ / COPY LICENSE NOTICE /unbounded/licenses/ RUN ldconfig From bb37e75ef36119e77142a00c5bf06a70a25ebe00 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 19:24:03 +0000 Subject: [PATCH 18/87] ci(racer): build the CUDA shim, emulate GPU kernels, and test racer-gpu --- .github/workflows/ci.yaml | 63 ++++++++ .../gpu/native/kernel_emulation.cc | 142 ++++++++++++++++++ .../gpu/native/run-kernel-emulation.sh | 17 +++ 3 files changed, 222 insertions(+) create mode 100644 cmd/racer-dataplane/gpu/native/kernel_emulation.cc create mode 100755 cmd/racer-dataplane/gpu/native/run-kernel-emulation.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b919ba578..ea9a42b33 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -355,6 +355,69 @@ jobs: if [ -n "$VERBS_FEATURES" ]; then features=(--features "$VERBS_FEATURES"); fi timeout --signal=TERM --kill-after=10s 300s cargo +1.96.0 test --locked --manifest-path cmd/racer-dataplane/Cargo.toml -p rdma-verbs --no-default-features "${features[@]}" + racer-gpu: + name: Racer GPU (${{ matrix.configuration }}) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - configuration: no-default-features + features: "" + native: false + - configuration: simulation + features: simulation + native: false + - configuration: native + features: native + native: true + - configuration: native-and-simulation + features: native,simulation + native: true + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/bin/racer-gpu-cargo + CARGO_BUILD_JOBS: "2" + GPU_FEATURES: ${{ matrix.features }} + LD_LIBRARY_PATH: ${{ github.workspace }}/bin/racer-gpu-native + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up pinned Rust + run: timeout --signal=TERM --kill-after=10s 300s rustup toolchain install 1.96.0 --profile minimal --component rustfmt,clippy + + # No CUDA toolkit on hosted runners: the shim builds without decrypt + # kernels and loads without a driver. Kernel math is covered by the + # C-core tests below and the kernel emulation step. + - name: Compile CUDA shim without kernels (not hardware validation) + if: matrix.native + run: timeout --signal=TERM --kill-after=10s 300s sh images/racer-dataplane/build-native-cuda.sh cmd/racer-dataplane/gpu/native bin/racer-gpu-native/libracer_gpu_cuda.so.1 + + - name: Require CUDA shim ABI and symbols (no hardware) + if: matrix.native + run: timeout --signal=TERM --kill-after=10s 300s cargo +1.96.0 test --locked --manifest-path cmd/racer-dataplane/Cargo.toml -p racer-gpu --no-default-features --features "$GPU_FEATURES" --lib ffi::tests::required_native_shim_loads_abi_and_symbols -- --exact --ignored + + - name: Run decrypt kernels on the CPU (kernel emulation) + if: matrix.native + run: timeout --signal=TERM --kill-after=10s 300s sh cmd/racer-dataplane/gpu/native/run-kernel-emulation.sh bin/racer-gpu-emulation + + - name: Check GPU formatting + run: timeout --signal=TERM --kill-after=10s 300s cargo +1.96.0 fmt --manifest-path cmd/racer-dataplane/Cargo.toml -p racer-gpu --check + + - name: Strict GPU Clippy + run: | + features=() + if [ -n "$GPU_FEATURES" ]; then features=(--features "$GPU_FEATURES"); fi + timeout --signal=TERM --kill-after=10s 300s cargo +1.96.0 clippy --locked --manifest-path cmd/racer-dataplane/Cargo.toml -p racer-gpu --all-targets --no-default-features "${features[@]}" -- -D warnings + + # Includes the C AES-GCM/CRC core check against racer-crypto (16 MiB pages). + - name: Test GPU crate without requiring GPU hardware + run: | + features=() + if [ -n "$GPU_FEATURES" ]; then features=(--features "$GPU_FEATURES"); fi + timeout --signal=TERM --kill-after=10s 300s cargo +1.96.0 test --locked --manifest-path cmd/racer-dataplane/Cargo.toml -p racer-gpu --no-default-features "${features[@]}" + runtime-miri: name: Runtime Miri (${{ matrix.group }}) runs-on: ubuntu-24.04 diff --git a/cmd/racer-dataplane/gpu/native/kernel_emulation.cc b/cmd/racer-dataplane/gpu/native/kernel_emulation.cc new file mode 100644 index 000000000..2a08d42e3 --- /dev/null +++ b/cmd/racer-dataplane/gpu/native/kernel_emulation.cc @@ -0,0 +1,142 @@ +// Runs the CUDA decrypt kernels (gcm_kernel.cu) on the CPU. +// +// Hosted CI has no GPU and no nvcc. This harness compiles the kernel source as +// C++ with one std::thread per CUDA thread, one block at a time, so +// __shared__ data is per block and __syncthreads is a real barrier. It uses +// the launch geometry from gpu_cuda.c and checks the kernels against the +// serial gcm_core.h path, which the Rust tests check against racer-crypto. +// +// Build and run: run-kernel-emulation.sh OUTPUT_DIR +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define __CUDACC__ 1 +#define __host__ +#define __device__ +#define __global__ +#define __shared__ static +struct emu_dim3 { + unsigned x, y, z; +}; +static thread_local emu_dim3 threadIdx; +static thread_local emu_dim3 blockIdx; +static thread_local emu_dim3 blockDim; +static std::barrier<> *emu_barrier; +static void __syncthreads() { emu_barrier->arrive_and_wait(); } + +#include "gcm_kernel.cu" + +static const uint8_t sbox[256] = GCM_SBOX_INIT; +static int failures; + +#define CHECK(cond, ...) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: ", __FILE__, __LINE__); \ + std::fprintf(stderr, __VA_ARGS__); \ + std::fprintf(stderr, "\n"); \ + ++failures; \ + } \ + } while (0) + +// Emulate <<>>. +static void launch(uint64_t grid, unsigned threads, const std::function &kernel) { + for (uint64_t b = 0; b < grid; ++b) { + auto barrier = std::make_unique>(threads); + emu_barrier = barrier.get(); + std::vector pool; + for (unsigned t = 0; t < threads; ++t) + pool.emplace_back([&, t] { + threadIdx = {t, 0, 0}; + blockIdx = {(unsigned)b, 0, 0}; + blockDim = {threads, 1, 1}; + kernel(); + }); + for (auto &thread : pool) thread.join(); + emu_barrier = nullptr; + } +} + +// The gpu_cuda.c sequence: partials, finalize, ctr. +static gcm_verdict run(const gcm_key &key, const std::vector &sealed, uint64_t n, + uint32_t check, uint64_t expected, std::vector &out) { + uint64_t chunks = gcm_chunks(n); + std::vector partials(chunks); + gcm_verdict verdict = {}; + launch((chunks + 127) / 128, 128, + [&] { racer_gcm_partials(sealed.data(), n, &key, partials.data(), chunks); }); + launch(1, 256, [&] { + racer_gcm_finalize(sealed.data(), n, &key, partials.data(), chunks, check, expected, &verdict); + }); + uint64_t blocks = (n + 15) / 16; + if (blocks) + launch((blocks + 255) / 256, 256, + [&] { racer_gcm_ctr(sealed.data(), n, &key, &verdict, out.data()); }); + return verdict; +} + +int main() { + uint64_t state = 0x243f6a8885a308d3ull; + auto next = [&] { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + return (uint8_t)state; + }; + uint64_t table[256]; + gcm_crc64_table(table); + for (uint64_t n : {0ull, 1ull, 15ull, 16ull, 17ull, 4079ull, 4080ull, 4081ull, 8192ull, + 20000ull, 70001ull}) { + uint8_t raw_key[32], nonce[12], aad[77]; + for (auto &b : raw_key) b = next(); + for (auto &b : nonce) b = next(); + for (auto &b : aad) b = next(); + std::vector plaintext(n), sealed(n + 16); + for (auto &b : plaintext) b = next(); + gcm_key key; + gcm_prepare(sbox, raw_key, nonce, aad, sizeof(aad), n, &key); + + // Seal: CTR is its own inverse, then the tag from the serial core. + for (uint64_t i = 0; i < (n + 15) / 16; ++i) gcm_ctr_block(sbox, &key, plaintext.data(), n, i, sealed.data()); + gcm_block s = gcm_load(key.prefix); + for (uint64_t j = 0; j < gcm_chunks(n); ++j) { + gcm_partial p; + gcm_chunk_partial(table, &key, sealed.data(), n, j, &p); + gcm_block g = gcm_load(p.ghash); + s.hi ^= g.hi; + s.lo ^= g.lo; + } + uint8_t tag[16]; + gcm_store(tag, s); + for (int i = 0; i < 16; ++i) sealed[n + i] = tag[i] ^ key.ekj0[i]; + uint64_t crc = gcm_crc64(table, sealed.data(), n + 16); + + std::vector out(n, 0xa5); + gcm_verdict v = run(key, sealed, n, 1, crc, out); + CHECK(v.status == GCM_VERDICT_PASS && v.crc == crc, "n=%llu pass status=%u", (unsigned long long)n, v.status); + CHECK(out == plaintext, "n=%llu plaintext", (unsigned long long)n); + + std::vector untouched(n, 0xa5); + out = untouched; + v = run(key, sealed, n, 1, crc ^ 1, out); + CHECK(v.status == GCM_VERDICT_CRC && out == untouched, "n=%llu crc", (unsigned long long)n); + + auto tampered = sealed; + tampered[n / 2] ^= 4; + v = run(key, tampered, n, 0, 0, out); + CHECK(v.status == GCM_VERDICT_TAG && out == untouched, "n=%llu tag", (unsigned long long)n); + } + if (failures) { + std::fprintf(stderr, "%d kernel emulation failures\n", failures); + return 1; + } + std::printf("kernel emulation passed\n"); + return 0; +} diff --git a/cmd/racer-dataplane/gpu/native/run-kernel-emulation.sh b/cmd/racer-dataplane/gpu/native/run-kernel-emulation.sh new file mode 100755 index 000000000..91d6abafb --- /dev/null +++ b/cmd/racer-dataplane/gpu/native/run-kernel-emulation.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +# +# Compile gcm_kernel.cu as host C++ (kernel_emulation.cc) and run it. +set -eu + +if [ "$#" -ne 1 ]; then + echo "usage: run-kernel-emulation.sh OUTPUT_DIR" >&2 + exit 2 +fi + +native=$(dirname "$0") +mkdir -p "$1" +"${CXX:-c++}" -std=c++20 -O2 -Wall -Wextra -Werror -pthread -I "$native" \ + -o "$1/kernel-emulation" "$native/kernel_emulation.cc" +"$1/kernel-emulation" From 24e83a3eec0256bc207f94cf4a1b2101a0f93cc5 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 19:24:20 +0000 Subject: [PATCH 19/87] docs(racer): clarify GPU AES timing note --- cmd/racer-dataplane/gpu/native/gcm_core.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/racer-dataplane/gpu/native/gcm_core.h b/cmd/racer-dataplane/gpu/native/gcm_core.h index 18451be32..048a9d3c4 100644 --- a/cmd/racer-dataplane/gpu/native/gcm_core.h +++ b/cmd/racer-dataplane/gpu/native/gcm_core.h @@ -31,7 +31,7 @@ * not constant time with respect to cache or shared-memory bank timing. The * kernels keep the S-box in shared memory, where only bank conflicts (not a * cache hierarchy shared with other tenants) can leak timing, and the key is a - * per-page derived key. A bitsliced AES would remove this; see the README. + * per-page derived key. A bitsliced AES would remove this; it is future work. */ #ifndef RACER_GCM_CORE_H #define RACER_GCM_CORE_H From 64241c66d04f3e29eb94245bdd0681f2084da68c Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 18:27:22 +0000 Subject: [PATCH 20/87] feat(racer): add io_uring RecvMsg with SCM_RIGHTS and shared positioned writes --- cmd/racer-dataplane/runtime/src/reactor.rs | 260 ++++++++++++++++++ .../runtime/src/reactor/tests.rs | 197 +++++++++++++ 2 files changed, 457 insertions(+) diff --git a/cmd/racer-dataplane/runtime/src/reactor.rs b/cmd/racer-dataplane/runtime/src/reactor.rs index 11499201f..af45cb84e 100644 --- a/cmd/racer-dataplane/runtime/src/reactor.rs +++ b/cmd/racer-dataplane/runtime/src/reactor.rs @@ -177,6 +177,140 @@ pub struct Completion { pub lease: L, } +/// Most `SCM_RIGHTS` descriptors one [`Reactor::recv_msg`] accepts. More set +/// [`MessageCompletion::truncated`], and the kernel closes the excess. +pub const MAX_RECEIVED_DESCRIPTORS: usize = 8; + +/// A fenced `recvmsg` result with any descriptors that arrived with the bytes. +pub struct MessageCompletion { + /// Original allocation, returned only after all applicable CQEs. + pub buffer: B, + + /// Bytes received, possibly less than the buffer length. + pub bytes: usize, + + /// Caller-owned admission or reuse guard retained through completion. + pub lease: L, + + /// Received `SCM_RIGHTS` descriptors, already close-on-exec. + pub descriptors: Vec, + + /// The kernel reported `MSG_CTRUNC`: ancillary data did not fit. + pub truncated: bool, +} + +/// Heap-pinned `msghdr`, `iovec`, and control storage for one `RecvMsg`. +/// Any descriptors the kernel installed are closed when this is dropped. +struct MessageHeader { + header: libc::msghdr, + + iov: libc::iovec, + + control: Vec, +} + +impl MessageHeader { + /// Control bytes able to hold [`MAX_RECEIVED_DESCRIPTORS`] descriptors. + fn control_bytes() -> usize { + // SAFETY: CMSG_SPACE only computes an aligned size. + unsafe { + libc::CMSG_SPACE((MAX_RECEIVED_DESCRIPTORS * std::mem::size_of::()) as u32) + as usize + } + } + + fn new() -> Box { + let words = Self::control_bytes().div_ceil(std::mem::size_of::()); + Box::new(Self { + // SAFETY: msghdr is plain old data; all-zero is a valid empty value. + header: unsafe { std::mem::zeroed() }, + iov: libc::iovec { + iov_base: std::ptr::null_mut(), + iov_len: 0, + }, + control: vec![0; words], + }) + } + + /// Point the header at `ptr..ptr+len` and an empty control buffer. + fn prepare(&mut self, ptr: *mut u8, len: u32) -> *mut libc::msghdr { + // Close anything left by an earlier attempt before reusing the storage. + drop(self.take_descriptors()); + self.control.fill(0); + self.iov = libc::iovec { + iov_base: ptr.cast(), + iov_len: len as usize, + }; + // SAFETY: as in new(). + self.header = unsafe { std::mem::zeroed() }; + self.header.msg_iov = &mut self.iov; + self.header.msg_iovlen = 1; + self.header.msg_control = self.control.as_mut_ptr().cast(); + self.header.msg_controllen = Self::control_bytes() as _; + &mut self.header + } + + /// Whether the kernel truncated ancillary data on the completed receive. + fn truncated(&self) -> bool { + self.header.msg_flags & libc::MSG_CTRUNC != 0 + } + + /// Take ownership of every `SCM_RIGHTS` descriptor in the control buffer. + /// The buffer is cleared, so a second call or the drop finds nothing. + // Control lengths are usize on glibc but narrower on other libcs. + #[allow(clippy::unnecessary_cast)] + fn take_descriptors(&mut self) -> Vec { + let mut descriptors = Vec::new(); + let capacity = self.control.len() * std::mem::size_of::(); + let used = (self.header.msg_controllen as usize).min(capacity); + if self.header.msg_control.is_null() || used < std::mem::size_of::() { + return descriptors; + } + let base = self.control.as_ptr().cast::(); + let mut offset = 0; + while offset + std::mem::size_of::() <= used { + // SAFETY: the header lies inside the initialized control buffer; + // read_unaligned tolerates any alignment. + let cmsg: libc::cmsghdr = unsafe { std::ptr::read_unaligned(base.add(offset).cast()) }; + let length = cmsg.cmsg_len as usize; + if length < std::mem::size_of::() || offset + length > used { + break; + } + // SAFETY: CMSG_LEN(0) is the aligned header size. + let data = unsafe { libc::CMSG_LEN(0) } as usize; + if cmsg.cmsg_level == libc::SOL_SOCKET && cmsg.cmsg_type == libc::SCM_RIGHTS { + let count = (length.saturating_sub(data)) / std::mem::size_of::(); + for index in 0..count { + // SAFETY: the descriptor array is inside this control message. + let fd: i32 = unsafe { + std::ptr::read_unaligned( + base.add(offset + data + index * std::mem::size_of::()) + .cast(), + ) + }; + if fd >= 0 { + // SAFETY: the kernel installed this descriptor for us alone. + descriptors.push(unsafe { HostFd::from_raw_fd(fd) }); + } + } + } + // CMSG_NXTHDR advances by CMSG_ALIGN(cmsg_len). + let step = length.next_multiple_of(std::mem::size_of::()); + offset += step; + } + self.control.fill(0); + self.header.msg_controllen = 0; + descriptors + } +} + +impl Drop for MessageHeader { + /// Close descriptors that no caller took, including after an abandoned wait. + fn drop(&mut self) { + drop(self.take_descriptors()); + } +} + /// An exclusively owned, stable backing allocation with an independent lifetime. /// /// # Safety @@ -1720,6 +1854,132 @@ impl Reactor { }) } + /// Receive like [`Self::recv`], also accepting `SCM_RIGHTS` descriptors with + /// `MSG_CMSG_CLOEXEC`. Descriptors are owned before this returns; a dropped + /// or failed wait closes any the kernel delivered. Simulated sockets carry + /// no ancillary data and always report none. + pub fn recv_msg<'a, B: IoBuffer, L: 'static>( + &'a self, + fd: Rc, + buffer: B, + lease: L, + scope: &'a S, + ) -> Operation<'a, MessageCompletion, S::Error> + where + S::Error: From, + { + Box::pin(async move { + let retained_fd = fd.clone(); + let ((owned, mut header), bytes) = self + .retry_data( + ( + InFlight { + file: fd, + buffer, + lease, + }, + MessageHeader::new(), + ), + retained_fd, + scope, + None, + libc::POLLIN as u32, + |(owned, header)| { + let (ptr, len) = + Self::buffer_region(&mut owned.buffer, BufferOperation::Recv)?; + let sqe = submission!( + self, + simulation::Op::Buffer { + fd: owned.file.clone(), + operation: BufferOperation::Recv, + ptr, + len: len as usize, + }, + opcode::RecvMsg::new( + types::Fd(owned.file.as_raw_fd()), + header.prepare(ptr, len) + ) + .flags(libc::MSG_CMSG_CLOEXEC as u32) + .build() + ); + Ok(sqe) + }, + ) + .await?; + let truncated = header.truncated(); + let descriptors = header.take_descriptors(); + let Completion { + buffer, + bytes, + lease, + } = owned.complete(bytes); + Ok(MessageCompletion { + buffer, + bytes, + lease, + descriptors, + truncated, + }) + }) + } + + /// Write immutable shared storage at an explicit file offset, returning + /// short progress without replaying bytes. Unlike [`Self::write_at`], the + /// buffer needs no mutable capability, so verified pages can be written. + pub fn write_shared_at<'a, B: SendBuffer, L: 'static>( + &'a self, + file: Rc, + offset: u64, + buffer: B, + lease: L, + scope: &'a S, + ) -> Operation<'a, Completion, S::Error> + where + S::Error: From, + { + Box::pin(async move { + if offset > i64::MAX as u64 { + return Err(Error::InvalidInput.into()); + } + let retained_fd = file.clone(); + let (owned, bytes) = self + .retry_data( + InFlight { + file, + buffer, + lease, + }, + retained_fd, + scope, + None, + libc::POLLOUT as u32, + |owned| { + let bytes = owned.buffer.send_bytes()?; + let len = u32::try_from(bytes.len()).map_err(|_| Error::InvalidInput)?; + let sqe = submission!( + self, + simulation::Op::Buffer { + fd: owned.file.clone(), + operation: BufferOperation::Write(offset), + ptr: bytes.as_ptr().cast_mut(), + len: len as usize, + }, + opcode::Write::new( + types::Fd(owned.file.as_raw_fd()), + bytes.as_ptr(), + len + ) + .offset(offset) + .build() + ); + Ok(sqe) + }, + ) + .await?; + Ok(owned.complete(bytes)) + }) + } + /// Select a readiness opcode without converting simulated owners to host FDs. fn readiness_submission(&self, fd: &Rc, interest: u32) -> Submission { submission!( diff --git a/cmd/racer-dataplane/runtime/src/reactor/tests.rs b/cmd/racer-dataplane/runtime/src/reactor/tests.rs index 3657dfcff..2f67cc363 100644 --- a/cmd/racer-dataplane/runtime/src/reactor/tests.rs +++ b/cmd/racer-dataplane/runtime/src/reactor/tests.rs @@ -3928,3 +3928,200 @@ mod table { assert!(table.slots.is_empty()); } } + +mod message { + //! `RecvMsg` descriptor passing and positioned shared-buffer writes. + use super::*; + use std::io::{Read, Write}; + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; + + /// Send `bytes` with `fds` as one `SCM_RIGHTS` message. + fn send_with_fds(socket: &UnixStream, bytes: &[u8], fds: &[RawFd]) { + let payload = std::mem::size_of_val(fds); + let space = unsafe { libc::CMSG_SPACE(payload as u32) } as usize; + let mut control = vec![0u64; space.div_ceil(8)]; + let mut iov = libc::iovec { + iov_base: bytes.as_ptr().cast_mut().cast(), + iov_len: bytes.len(), + }; + let mut header: libc::msghdr = unsafe { std::mem::zeroed() }; + header.msg_iov = &mut iov; + header.msg_iovlen = 1; + if !fds.is_empty() { + header.msg_control = control.as_mut_ptr().cast(); + header.msg_controllen = space as _; + unsafe { + let cmsg = libc::CMSG_FIRSTHDR(&header); + (*cmsg).cmsg_level = libc::SOL_SOCKET; + (*cmsg).cmsg_type = libc::SCM_RIGHTS; + (*cmsg).cmsg_len = libc::CMSG_LEN(payload as u32) as _; + std::ptr::copy_nonoverlapping( + fds.as_ptr().cast::(), + libc::CMSG_DATA(cmsg), + payload, + ); + } + } + let sent = unsafe { libc::sendmsg(socket.as_raw_fd(), &header, 0) }; + assert_eq!( + sent, + bytes.len() as isize, + "{}", + std::io::Error::last_os_error() + ); + } + + fn pipe() -> (std::fs::File, OwnedFd) { + let mut fds = [0; 2]; + assert_eq!(unsafe { libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC) }, 0); + unsafe { + ( + std::fs::File::from_raw_fd(fds[0]), + OwnedFd::from_raw_fd(fds[1]), + ) + } + } + + fn memfd() -> std::fs::File { + let fd = unsafe { libc::memfd_create(c"racer-test".as_ptr(), libc::MFD_CLOEXEC) }; + assert!(fd >= 0); + unsafe { std::fs::File::from_raw_fd(fd) } + } + + fn cloexec(fd: &OwnedFd) -> bool { + unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFD) & libc::FD_CLOEXEC != 0 } + } + + #[test] + /// One descriptor arrives owned and close-on-exec with its bytes. + fn receives_one_descriptor_with_bytes() { + let Some(reactor) = kernel_reactor(4) else { + return; + }; + let request = scope(); + let (socket, peer) = UnixStream::pair().unwrap(); + socket.set_nonblocking(true).unwrap(); + let mut file = memfd(); + send_with_fds(&peer, b"head", &[file.as_raw_fd()]); + let drops = Rc::new(Cell::new(0)); + let completed = drive( + &reactor, + reactor.recv_msg( + Rc::new(Descriptor::from(socket)), + Buffer(vec![0; 16], drops.clone()), + Lease(drops.clone()), + &request, + ), + ) + .unwrap(); + assert_eq!(completed.bytes, 4); + assert_eq!(&completed.buffer.0[..4], b"head"); + assert!(!completed.truncated); + assert_eq!(completed.descriptors.len(), 1); + assert!(cloexec(&completed.descriptors[0])); + let mut received = std::fs::File::from(completed.descriptors.into_iter().next().unwrap()); + received.write_all(b"shared").unwrap(); + let mut output = String::new(); + use std::io::Seek; + file.seek(std::io::SeekFrom::Start(0)).unwrap(); + file.read_to_string(&mut output).unwrap(); + assert_eq!(output, "shared"); + } + + #[test] + /// Plain bytes report no descriptors; too many descriptors set truncation. + fn plain_bytes_and_truncated_control_data() { + let Some(reactor) = kernel_reactor(4) else { + return; + }; + let request = scope(); + let (socket, peer) = UnixStream::pair().unwrap(); + socket.set_nonblocking(true).unwrap(); + let socket = Rc::new(Descriptor::from(socket)); + (&peer).write_all(b"x").unwrap(); + let completed = drive( + &reactor, + reactor.recv_msg(socket.clone(), buffer(&[0; 4]), (), &request), + ) + .unwrap(); + assert_eq!(completed.bytes, 1); + assert!(completed.descriptors.is_empty()); + assert!(!completed.truncated); + + let (mut read, write) = pipe(); + let many = vec![write.as_raw_fd(); MAX_RECEIVED_DESCRIPTORS + 1]; + send_with_fds(&peer, b"y", &many); + drop(write); + let completed = drive( + &reactor, + reactor.recv_msg(socket, buffer(&[0; 4]), (), &request), + ) + .unwrap(); + assert_eq!(completed.bytes, 1); + assert!(completed.truncated); + assert!(completed.descriptors.len() <= MAX_RECEIVED_DESCRIPTORS); + drop(completed); + // Every copy of the write end is closed: no descriptor leaked. + let mut byte = [0]; + assert_eq!(read.read(&mut byte).unwrap(), 0); + } + + #[test] + /// A wait dropped before its CQE still closes the delivered descriptor. + fn abandoned_receive_closes_descriptors() { + let Some(reactor) = kernel_reactor(4) else { + return; + }; + let request = scope(); + let (socket, peer) = UnixStream::pair().unwrap(); + socket.set_nonblocking(true).unwrap(); + let drops = Rc::new(Cell::new(0)); + let mut receive = reactor.recv_msg( + Rc::new(Descriptor::from(socket)), + Buffer(vec![0; 4], drops.clone()), + Lease(drops.clone()), + &request, + ); + assert!(poll(&mut receive).is_pending()); + reactor.wait(Duration::ZERO).unwrap(); + drop(receive); + let (mut read, write) = pipe(); + send_with_fds(&peer, b"z", &[write.as_raw_fd()]); + drop(write); + let deadline = Instant::now() + Duration::from_secs(5); + while drops.get() != 2 { + assert!(Instant::now() < deadline, "abandoned receive never fenced"); + reactor.poll_budgeted(8).unwrap(); + reactor.wait(Duration::from_millis(1)).unwrap(); + } + let mut byte = [0]; + assert_eq!(read.read(&mut byte).unwrap(), 0); + } + + #[test] + /// Positioned shared writes land at their offset and reject huge offsets. + fn shared_write_at_offset() { + let Some(reactor) = kernel_reactor(4) else { + return; + }; + let request = scope(); + let mut file = memfd(); + let fd = Rc::new(Descriptor::from(OwnedFd::from(file.try_clone().unwrap()))); + let completed = drive( + &reactor, + reactor.write_shared_at(fd.clone(), 3, buffer(b"abc"), (), &request), + ) + .unwrap(); + assert_eq!(completed.bytes, 3); + let mut output = Vec::new(); + file.read_to_end(&mut output).unwrap(); + assert_eq!(output, b"\0\0\0abc"); + assert!(matches!( + drive( + &reactor, + reactor.write_shared_at(fd, i64::MAX as u64 + 1, buffer(b"a"), (), &request), + ), + Err(Error::InvalidInput) + )); + } +} From 270f015cde2e6c87d4054171dcb3c62f2b8ecbfd Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 18:30:22 +0000 Subject: [PATCH 21/87] feat(racer): parse host-read and gpu-read requests and build direct heads --- cmd/racer-dataplane/object-wire/src/client.rs | 217 ++++++++++++-- .../object-wire/tests/direct.rs | 283 ++++++++++++++++++ 2 files changed, 478 insertions(+), 22 deletions(-) create mode 100644 cmd/racer-dataplane/object-wire/tests/direct.rs diff --git a/cmd/racer-dataplane/object-wire/src/client.rs b/cmd/racer-dataplane/object-wire/src/client.rs index 81fd1abe1..fd060280a 100644 --- a/cmd/racer-dataplane/object-wire/src/client.rs +++ b/cmd/racer-dataplane/object-wire/src/client.rs @@ -2,7 +2,7 @@ use crate::model::{ ByteRange, CacheKey, MAX_FIELD_BYTES, MAX_WIRE_INTEGER, ObjectMetadata, PAGE_BYTES, - ResolvedRange, StrongEtag, + ResolvedRange, StrongEtag, parse_decimal, }; use crate::{Error, Result}; use http1::{Header, MessageHead, StartLine, is_token, trim_ows}; @@ -28,6 +28,58 @@ pub enum ReadKind { byte_credits: u64, }, + /// Write the plaintext into a client file instead of the socket. + HostRead { + pin: Option, + + range: Option, + + destination: Destination, + }, + /// Write the plaintext into a client GPU allocation. + GpuRead { + pin: Option, + + range: Option, + + destination: Destination, + + /// The size the client passed to `cuMemCreate`. + allocation_size: u64, + }, +} + +/// The client window `[offset, offset + length)` that a direct read may write. +/// The parser guarantees `length > 0` and that the end fits in a wire integer. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Destination { + pub offset: u64, + + pub length: u64, +} + +impl Destination { + /// Exclusive end of the window. + pub fn end(&self) -> u64 { + self.offset + self.length + } +} + +/// Where a direct read puts the plaintext, sent as `Racer-Delivery`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DirectDelivery { + Host, + Gpu, +} + +impl DirectDelivery { + /// The `Racer-Delivery` header value. + pub fn as_str(self) -> &'static str { + match self { + Self::Host => "host", + Self::Gpu => "gpu", + } + } } impl ReadKind { @@ -40,7 +92,39 @@ impl ReadKind { pub fn pin(&self) -> Option<&StrongEtag> { match self { Self::HeadPinned { etag } => Some(etag), - Self::Subscription { pin, .. } => pin.as_ref(), + Self::Subscription { pin, .. } + | Self::HostRead { pin, .. } + | Self::GpuRead { pin, .. } => pin.as_ref(), + _ => None, + } + } + + /// The requested range of a body read, if any. + pub fn range(&self) -> Option { + match self { + Self::Subscription { range, .. } + | Self::HostRead { range, .. } + | Self::GpuRead { range, .. } => *range, + _ => None, + } + } + + /// Whether this request writes into a client descriptor, not the socket. + /// Exactly these requests must carry one descriptor. + pub fn direct(&self) -> Option { + match self { + Self::HostRead { .. } => Some(DirectDelivery::Host), + Self::GpuRead { .. } => Some(DirectDelivery::Gpu), + _ => None, + } + } + + /// The destination window of a direct read. + pub fn destination(&self) -> Option { + match self { + Self::HostRead { destination, .. } | Self::GpuRead { destination, .. } => { + Some(*destination) + } _ => None, } } @@ -86,6 +170,19 @@ impl RequestParser { return Err(Error::InvalidRequest); }; let mut decoded_bytes = method.len().saturating_add(target.len()); + let path = target + .strip_prefix("/v1/objects/") + .ok_or(Error::InvalidRequest)?; + // Only the exact suffixes select a direct read; anything else keeps the + // plain object path and its existing key validation. + let (key_text, direct) = match path.split_once('/') { + Some((key, "host-read")) => (key, Some(DirectDelivery::Host)), + Some((key, "gpu-read")) => (key, Some(DirectDelivery::Gpu)), + _ => (path, None), + }; + let mut destination_offset = None; + let mut destination_length = None; + let mut allocation_size = None; let mut seen = HashSet::new(); let mut host = None; let mut pin = None; @@ -165,8 +262,19 @@ impl RequestParser { pin = Some(StrongEtag::parse(value)?); } "range" => range = Some(ByteRange::parse(value)?), + // Direct reads are paced by the dataplane and ignore credits. + "racer-page-credits" | "racer-byte-credits" if direct.is_some() => {} "racer-page-credits" => page_credits = decimal(value, 1, 64)? as usize, "racer-byte-credits" => byte_credits = decimal(value, PAGE_BYTES, 64 * PAGE_BYTES)?, + "racer-destination-offset" if direct.is_some() => { + once(&mut destination_offset, parse_decimal(value)?)?; + } + "racer-destination-length" if direct.is_some() => { + once(&mut destination_length, parse_decimal(value)?)?; + } + "racer-allocation-size" if direct == Some(DirectDelivery::Gpu) => { + once(&mut allocation_size, parse_decimal(value)?)?; + } "racer-metadata" => { validate_opaque(value)?; metadata = Some(value); @@ -184,26 +292,56 @@ impl RequestParser { if host != Some(true) { return Err(Error::InvalidRequest); } - let key = CacheKey::parse_hex( - target - .strip_prefix("/v1/objects/") - .ok_or(Error::InvalidRequest)? - .as_bytes(), - )?; - let kind = match method.as_str() { - "HEAD" if range.is_none() => match pin { - Some(etag) => ReadKind::HeadPinned { etag }, - None => ReadKind::Head, - }, - "HEAD" => return Err(Error::InvalidRequest), - "POST" if content_length => ReadKind::Subscription { - pin, - range, - page_credits, - byte_credits, + let key = CacheKey::parse_hex(key_text.as_bytes())?; + let kind = match (method.as_str(), direct) { + ("POST", Some(_)) if !content_length => return Err(Error::InvalidRequest), + ("POST", Some(delivery)) => { + let destination = Destination { + offset: destination_offset.ok_or(Error::InvalidRequest)?, + length: destination_length + .filter(|&length| length != 0) + .ok_or(Error::InvalidRequest)?, + }; + destination + .offset + .checked_add(destination.length) + .filter(|&end| end <= MAX_WIRE_INTEGER) + .ok_or(Error::InvalidRequest)?; + match delivery { + DirectDelivery::Host => ReadKind::HostRead { + pin, + range, + destination, + }, + DirectDelivery::Gpu => { + let allocation_size = allocation_size + .filter(|&size| destination.end() <= size) + .ok_or(Error::InvalidRequest)?; + ReadKind::GpuRead { + pin, + range, + destination, + allocation_size, + } + } + } + } + (_, Some(_)) => return Err(Error::MethodNotAllowed), + (method, None) => match method { + "HEAD" if range.is_none() => match pin { + Some(etag) => ReadKind::HeadPinned { etag }, + None => ReadKind::Head, + }, + "HEAD" => return Err(Error::InvalidRequest), + "POST" if content_length => ReadKind::Subscription { + pin, + range, + page_credits, + byte_credits, + }, + "POST" => return Err(Error::InvalidRequest), + _ => return Err(Error::MethodNotAllowed), }, - "POST" => return Err(Error::InvalidRequest), - _ => return Err(Error::MethodNotAllowed), }; Ok(ClientRequest { kind, @@ -271,6 +409,14 @@ fn parse_max_age(value: &[u8], max_age: &mut Option) -> Result<()> { Ok(()) } +/// Store a destination field, rejecting a repeated one. +fn once(slot: &mut Option, value: u64) -> Result<()> { + if slot.replace(value).is_some() { + return Err(Error::InvalidRequest); + } + Ok(()) +} + /// Parse a nonzero canonical credit value inside its protocol bounds. fn decimal(value: &[u8], minimum: u64, maximum: u64) -> Result { if value.is_empty() || value[0] == b'0' || !value.iter().all(u8::is_ascii_digit) { @@ -307,6 +453,15 @@ fn validate_opaque(value: &[u8]) -> Result<()> { pub fn subscription_head( metadata: &ObjectMetadata, range: Option, +) -> Result { + body_head(metadata, range, true) +} + +/// Shared Get-style head. Without payload, page frames carry only their 21 bytes. +fn body_head( + metadata: &ObjectMetadata, + range: Option, + payload: bool, ) -> Result { let mut head = success_head(metadata, None)?; head.headers @@ -323,7 +478,7 @@ pub fn subscription_head( let length = pages .checked_add(1) .and_then(|n| n.checked_mul(21)) - .and_then(|n| n.checked_add(end - start)) + .and_then(|n| n.checked_add(if payload { end - start } else { 0 })) .filter(|&n| n <= MAX_WIRE_INTEGER) .ok_or(Error::BadGateway)?; head.start = StartLine::Response { status: 200 }; @@ -337,6 +492,24 @@ pub fn subscription_head( Ok(head) } +/// Encode a direct-read success head. Page frames carry no payload, so the +/// body is `21 * (pages + 1)` bytes. An empty object has no pages. +pub fn direct_head( + metadata: &ObjectMetadata, + range: Option, + delivery: DirectDelivery, +) -> Result { + let mut head = body_head(metadata, range, false)?; + let close = head + .headers + .iter() + .position(|h| h.name.eq_ignore_ascii_case("Connection")) + .unwrap_or(head.headers.len()); + head.headers + .insert(close, header("Racer-Delivery", delivery.as_str())); + Ok(head) +} + /// Encode the fixed-width page or completion frame without managing page leases. pub fn frame(kind: u8, number: u64, offset: u64, length: u32) -> [u8; 21] { let mut bytes = [0; 21]; diff --git a/cmd/racer-dataplane/object-wire/tests/direct.rs b/cmd/racer-dataplane/object-wire/tests/direct.rs new file mode 100644 index 000000000..8e2e5fe91 --- /dev/null +++ b/cmd/racer-dataplane/object-wire/tests/direct.rs @@ -0,0 +1,283 @@ +//! Direct-read (`host-read`, `gpu-read`) request parsing and response heads. + +use http1::{Header, MessageHead, StartLine}; +use racer_control_wire::CacheId; +use racer_object_wire::client::{self, Destination, DirectDelivery, ReadKind, RequestParser}; +use racer_object_wire::*; + +fn key() -> String { + CacheKey([0xab; 32]).to_hex() +} + +/// A direct request with valid destination fields plus `extra`. +fn direct(method: &str, suffix: &str, extra: &[(&str, &str)]) -> MessageHead { + let mut headers = vec![ + Header::new("Host", "racer"), + Header::new("Content-Length", "0"), + ]; + headers.extend(extra.iter().map(|(name, value)| Header::new(*name, *value))); + MessageHead { + start: StartLine::Request { + method: method.into(), + target: format!("/v1/objects/{}/{suffix}", key()), + }, + headers, + } +} + +fn parse(head: &MessageHead) -> Result { + RequestParser::new(usize::MAX) + .parse(head) + .map(|request| request.kind) +} + +const DEST: [(&str, &str); 2] = [ + ("Racer-Destination-Offset", "4096"), + ("Racer-Destination-Length", "100"), +]; + +#[test] +fn host_read_parses_range_pin_and_destination() { + let head = direct( + "POST", + "host-read", + &[ + DEST[0], + DEST[1], + ("Range", "bytes=1-3"), + ("If-Match", "\"v1\""), + // Credits are ignored, even when they would be invalid on a Get. + ("Racer-Page-Credits", "0"), + ("Racer-Byte-Credits", "x"), + ("Racer-Allocation-Size", "junk"), + ], + ); + let request = RequestParser::new(usize::MAX).parse(&head).unwrap(); + assert_eq!(request.key, CacheKey([0xab; 32])); + let kind = request.kind; + assert_eq!( + kind, + ReadKind::HostRead { + pin: Some(StrongEtag::parse(b"\"v1\"").unwrap()), + range: Some(ByteRange::closed(1, 3).unwrap()), + destination: Destination { + offset: 4096, + length: 100 + }, + } + ); + assert_eq!(kind.direct(), Some(DirectDelivery::Host)); + assert_eq!(kind.destination().unwrap().end(), 4196); + assert_eq!(kind.range(), Some(ByteRange::closed(1, 3).unwrap())); + assert!(kind.pin().is_some()); + assert!(!kind.is_head()); +} + +#[test] +fn gpu_read_requires_allocation_that_holds_the_window() { + let parsed = parse(&direct( + "POST", + "gpu-read", + &[DEST[0], DEST[1], ("Racer-Allocation-Size", "4196")], + )) + .unwrap(); + assert_eq!( + parsed, + ReadKind::GpuRead { + pin: None, + range: None, + destination: Destination { + offset: 4096, + length: 100 + }, + allocation_size: 4196, + } + ); + assert_eq!(parsed.direct(), Some(DirectDelivery::Gpu)); + for extra in [ + &[DEST[0], DEST[1]][..], + &[DEST[0], DEST[1], ("Racer-Allocation-Size", "4195")], + &[DEST[0], DEST[1], ("Racer-Allocation-Size", "-1")], + &[ + DEST[0], + DEST[1], + ("Racer-Allocation-Size", "9999"), + ("Racer-Allocation-Size", "9999"), + ], + ] { + assert_eq!( + parse(&direct("POST", "gpu-read", extra)), + Err(Error::InvalidRequest), + "{extra:?}" + ); + } +} + +#[test] +fn destination_fields_are_required_canonical_unique_and_bounded() { + let max = i64::MAX.to_string(); + for suffix in ["host-read", "gpu-read"] { + let size = ("Racer-Allocation-Size", max.as_str()); + for extra in [ + &[size][..], + &[size, DEST[0]], + &[size, DEST[1]], + &[size, DEST[0], ("Racer-Destination-Length", "0")], + &[size, DEST[0], ("Racer-Destination-Length", "01")], + &[size, DEST[1], ("Racer-Destination-Offset", " 1")], + &[ + size, + DEST[1], + ("Racer-Destination-Offset", "9223372036854775808"), + ], + &[size, DEST[0], DEST[1], DEST[1]], + &[size, DEST[0], DEST[0], DEST[1]], + // offset + length must stay inside a wire integer. + &[ + size, + ("Racer-Destination-Offset", max.as_str()), + ("Racer-Destination-Length", "1"), + ], + ] { + assert_eq!( + parse(&direct("POST", suffix, extra)), + Err(Error::InvalidRequest), + "{suffix} {extra:?}" + ); + } + let ok = parse(&direct( + "POST", + suffix, + &[ + size, + ("Racer-Destination-Offset", "0"), + ("Racer-Destination-Length", max.as_str()), + ], + )) + .unwrap(); + assert_eq!(ok.destination().unwrap().end(), i64::MAX as u64); + } +} + +#[test] +fn direct_paths_require_post_with_zero_length_and_known_suffix() { + for method in ["GET", "HEAD", "PUT"] { + assert_eq!( + parse(&direct(method, "host-read", &DEST)), + Err(Error::MethodNotAllowed) + ); + } + let mut head = direct("POST", "host-read", &DEST); + head.headers.retain(|h| h.name != "Content-Length"); + assert_eq!(parse(&head), Err(Error::InvalidRequest)); + for suffix in ["cpu-read", "host-read/", "", "HOST-READ"] { + assert_eq!( + parse(&direct("POST", suffix, &DEST)), + Err(Error::InvalidRequest), + "{suffix}" + ); + } + // The key is still validated on direct paths. + let mut head = direct("POST", "host-read", &DEST); + head.start = StartLine::Request { + method: "POST".into(), + target: format!("/v1/objects/{}/host-read", "A".repeat(64)), + }; + assert_eq!(parse(&head), Err(Error::InvalidRequest)); +} + +#[test] +fn plain_requests_ignore_destination_fields_as_before() { + let mut head = direct("POST", "", &[]); + head.start = StartLine::Request { + method: "POST".into(), + target: format!("/v1/objects/{}", key()), + }; + let plain = parse(&head).unwrap(); + head.headers.extend([ + Header::new("Racer-Destination-Offset", "x"), + Header::new("Racer-Destination-Offset", "y"), + Header::new("Racer-Allocation-Size", "z"), + ]); + assert_eq!(parse(&head).unwrap(), plain); + assert!(matches!(plain, ReadKind::Subscription { .. })); + assert_eq!(plain.direct(), None); + assert_eq!(plain.destination(), None); +} + +fn metadata(length: u64) -> ObjectMetadata { + ObjectMetadata { + content_type: None, + version: ObjectVersion { + object: ObjectId { + cache: CacheId("cache".into()), + key: CacheKey([0xab; 32]), + }, + etag: StrongEtag::parse(b"\"v1\"").unwrap(), + }, + length, + expires_at: ExpiresAt::from_unix_millis(1234).unwrap(), + } +} + +#[test] +fn direct_head_counts_only_frames_and_names_delivery() { + let length = 2 * PAGE_BYTES + 5; + for (range, delivery, pages) in [ + (Some((0, length)), DirectDelivery::Host, 3), + ( + Some((PAGE_BYTES - 1, PAGE_BYTES + 1)), + DirectDelivery::Gpu, + 2, + ), + (Some((3, 4)), DirectDelivery::Host, 1), + ] { + let range = range.map(|(start, end)| { + ByteRange::closed(start, end - 1) + .unwrap() + .resolve(length) + .unwrap() + }); + let head = client::direct_head(&metadata(length), range, delivery).unwrap(); + let subscription = client::subscription_head(&metadata(length), range).unwrap(); + assert!(matches!(head.start, StartLine::Response { status: 200 })); + assert_eq!( + head.unique("Content-Length").unwrap(), + Some((21 * (pages + 1)).to_string().as_bytes()) + ); + assert_eq!( + head.unique("Racer-Delivery").unwrap(), + Some(delivery.as_str().as_bytes()) + ); + assert_eq!( + head.unique("Connection").unwrap(), + Some(b"close".as_slice()) + ); + // Every other field matches the Get head. + for field in &subscription.headers { + if !field.name.eq_ignore_ascii_case("Content-Length") { + assert_eq!( + head.unique(&field.name).unwrap(), + Some(field.value.as_slice()) + ); + } + } + assert_eq!(head.headers.len(), subscription.headers.len() + 1); + } + let empty = client::direct_head(&metadata(0), None, DirectDelivery::Host).unwrap(); + assert_eq!( + empty.unique("Content-Length").unwrap(), + Some(b"21".as_slice()) + ); + // A maximum-length object fits because frames carry no payload. + let range = ByteRange::From(0).resolve(MAX_WIRE_INTEGER).unwrap(); + assert!( + client::direct_head( + &metadata(MAX_WIRE_INTEGER), + Some(range), + DirectDelivery::Host + ) + .is_ok() + ); + assert!(client::subscription_head(&metadata(MAX_WIRE_INTEGER), Some(range)).is_err()); +} From 83dffbabccc7b804fae2eeef88bbd23211ae56bf Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 18:31:32 +0000 Subject: [PATCH 22/87] feat(racer): collect SCM_RIGHTS descriptors while reading request heads --- cmd/racer-dataplane/http/src/connection.rs | 78 ++++++++++++++++--- cmd/racer-dataplane/http/tests/exchanges.rs | 83 +++++++++++++++++++++ 2 files changed, 150 insertions(+), 11 deletions(-) diff --git a/cmd/racer-dataplane/http/src/connection.rs b/cmd/racer-dataplane/http/src/connection.rs index 52f802b35..c163f05be 100644 --- a/cmd/racer-dataplane/http/src/connection.rs +++ b/cmd/racer-dataplane/http/src/connection.rs @@ -321,6 +321,25 @@ pub struct HeadCompletion { pub _decoded: Option, } +/// Descriptors passed with `SCM_RIGHTS` while a request head was read. +#[derive(Debug, Default)] +pub struct ReceivedDescriptors { + /// Received descriptors, close-on-exec, in arrival order. + pub descriptors: Vec, + + /// Some ancillary data was truncated (`MSG_CTRUNC`) and its descriptors lost. + pub truncated: bool, +} + +/// A decoded request head and the descriptors that arrived with it. +pub struct RequestHead { + /// Admitted request head. + pub head: MessageHead, + + /// Descriptors received while reading this head. + pub descriptors: ReceivedDescriptors, +} + /// Fixed admitted storage, zeroized before release or single-buffer reuse. pub struct OwnedBuffer { bytes: Vec, @@ -557,7 +576,7 @@ impl HttpIo { ) -> Operation<'a, C, HeadCompletion> { Box::pin(async move { let completed = self - .receive_head_outcome(connection, scope, header_limit, false) + .receive_head_outcome(connection, scope, header_limit, false, None) .await?; Ok(HeadCompletion { connection: completed.connection, @@ -576,7 +595,33 @@ impl HttpIo { scope: &'a C::Scope, header_limit: usize, ) -> Operation<'a, C, HeadCompletion>> { - self.receive_head_outcome(connection, scope, header_limit, true) + self.receive_head_outcome(connection, scope, header_limit, true, None) + } + + /// Like [`Self::receive_request_head_limited`], but read with `recvmsg` and + /// keep any `SCM_RIGHTS` descriptors that arrived while reading the head. + /// Descriptors that came with a rejected head are closed. The caller owns + /// the returned descriptors and must close any the request does not claim. + pub fn receive_request_head_with_descriptors<'a>( + &'a self, + connection: ConnectionLease, + scope: &'a C::Scope, + header_limit: usize, + ) -> Operation<'a, C, HeadCompletion>> { + Box::pin(async move { + let sink = Rc::new(RefCell::new(ReceivedDescriptors::default())); + let completed = self + .receive_head_outcome(connection, scope, header_limit, true, Some(sink.clone())) + .await?; + let descriptors = std::mem::take(&mut *sink.borrow_mut()); + Ok(HeadCompletion { + connection: completed.connection, + value: completed + .value + .map(|head| RequestHead { head, descriptors }), + _decoded: completed._decoded, + }) + }) } /// Decode and admit a head without allowing hooks to change wire framing. @@ -586,6 +631,7 @@ impl HttpIo { scope: &'a C::Scope, header_limit: usize, request_only: bool, + ancillary: Option>>, ) -> Operation<'a, C, HeadCompletion>> { Box::pin(async move { scope.check()?; @@ -670,15 +716,25 @@ impl HttpIo { buffer = larger; } let end = buffer.bytes.len(); - let completed = self - .reactor - .recv( - connection.socket(), - BufferRange::new::(buffer, used..end)?, - connection, - scope, - ) - .await?; + let range = BufferRange::new::(buffer, used..end)?; + let completed = if let Some(sink) = &ancillary { + let received = self + .reactor + .recv_msg(connection.socket(), range, connection, scope) + .await?; + let mut sink = sink.borrow_mut(); + sink.truncated |= received.truncated; + sink.descriptors.extend(received.descriptors); + Completion { + buffer: received.buffer, + bytes: received.bytes, + lease: received.lease, + } + } else { + self.reactor + .recv(connection.socket(), range, connection, scope) + .await? + }; if completed.bytes == 0 || completed.bytes > end - used { return Err(uring_runtime::Error::Io.into()); } diff --git a/cmd/racer-dataplane/http/tests/exchanges.rs b/cmd/racer-dataplane/http/tests/exchanges.rs index 9c72f253b..b72921133 100644 --- a/cmd/racer-dataplane/http/tests/exchanges.rs +++ b/cmd/racer-dataplane/http/tests/exchanges.rs @@ -685,3 +685,86 @@ fn response_on_request_only_connection_can_be_rejected_but_never_reused() { assert!(!response.connection.is_reusable()); assert_eq!(io.reactor().in_flight(), 0); } + +/// Send `bytes` with `fds` as one `SCM_RIGHTS` message on a raw stream. +fn send_with_fds(socket: &UnixStream, bytes: &[u8], fds: &[std::os::fd::RawFd]) { + use std::os::fd::AsRawFd; + let payload = std::mem::size_of_val(fds); + let space = unsafe { libc::CMSG_SPACE(payload as u32) } as usize; + let mut control = vec![0u64; space.div_ceil(8)]; + let mut iov = libc::iovec { + iov_base: bytes.as_ptr().cast_mut().cast(), + iov_len: bytes.len(), + }; + let mut header: libc::msghdr = unsafe { std::mem::zeroed() }; + header.msg_iov = &mut iov; + header.msg_iovlen = 1; + header.msg_control = control.as_mut_ptr().cast(); + header.msg_controllen = space as _; + unsafe { + let cmsg = libc::CMSG_FIRSTHDR(&header); + (*cmsg).cmsg_level = libc::SOL_SOCKET; + (*cmsg).cmsg_type = libc::SCM_RIGHTS; + (*cmsg).cmsg_len = libc::CMSG_LEN(payload as u32) as _; + std::ptr::copy_nonoverlapping(fds.as_ptr().cast::(), libc::CMSG_DATA(cmsg), payload); + } + let sent = unsafe { libc::sendmsg(socket.as_raw_fd(), &header, 0) }; + assert_eq!(sent, bytes.len() as isize); +} + +/// Descriptors sent with any part of a request head are returned with it, and +/// a rejected head closes them. +#[test] +fn request_heads_collect_passed_descriptors() { + use std::io::{Read, Write}; + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + let io = io(); + for malformed in [false, true] { + let (mut client, server) = UnixStream::pair().unwrap(); + let server = ConnectionLease::from_reserved(server.into(), (), ()).unwrap(); + let mut fds = [0; 2]; + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + let mut read = unsafe { std::fs::File::from_raw_fd(fds[0]) }; + let write = unsafe { OwnedFd::from_raw_fd(fds[1]) }; + let first: &[u8] = if malformed { + b"BAD\r\n" + } else { + b"POST /x HTTP/1.1\r\n" + }; + send_with_fds(&client, first, &[write.as_raw_fd(), write.as_raw_fd()]); + drop(write); + client.write_all(b"Content-Length: 0\r\n\r\n").unwrap(); + let received = drive( + &io, + io.receive_request_head_with_descriptors(server, &RequestScope, 128), + ) + .unwrap(); + let mut byte = [0]; + if malformed { + assert!(received.value.is_err()); + } else { + let head = received.value.unwrap(); + assert!(matches!(head.head.start, StartLine::Request { .. })); + assert!(!head.descriptors.truncated); + assert_eq!(head.descriptors.descriptors.len(), 2); + drop(head); + } + // Every copy is closed: the pipe reports EOF. + assert_eq!(read.read(&mut byte).unwrap(), 0); + } + // A plain head reports no descriptors. + let (mut client, server) = UnixStream::pair().unwrap(); + let server = ConnectionLease::from_reserved(server.into(), (), ()).unwrap(); + client + .write_all(b"POST /x HTTP/1.1\r\nContent-Length: 0\r\n\r\n") + .unwrap(); + let head = drive( + &io, + io.receive_request_head_with_descriptors(server, &RequestScope, 128), + ) + .unwrap() + .value + .unwrap(); + assert!(head.descriptors.descriptors.is_empty()); + assert!(!head.descriptors.truncated); +} From 3a232ab34d62513f45f7678fcca8d329296f7bc8 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 18:37:06 +0000 Subject: [PATCH 23/87] feat(racer): serve host-read and add the gpu-read sink seam --- cmd/racer-dataplane/src/client.rs | 221 ++++++++++++++++++++- cmd/racer-dataplane/src/client/direct.rs | 208 +++++++++++++++++++ cmd/racer-dataplane/src/client/listener.rs | 14 +- cmd/racer-dataplane/src/client/tests.rs | 1 + cmd/racer-dataplane/src/error.rs | 2 + cmd/racer-dataplane/src/http.rs | 43 ++++ cmd/racer-dataplane/src/read.rs | 20 +- 7 files changed, 504 insertions(+), 5 deletions(-) create mode 100644 cmd/racer-dataplane/src/client/direct.rs diff --git a/cmd/racer-dataplane/src/client.rs b/cmd/racer-dataplane/src/client.rs index c0dcacc40..8c880e5b9 100644 --- a/cmd/racer-dataplane/src/client.rs +++ b/cmd/racer-dataplane/src/client.rs @@ -41,9 +41,15 @@ use std::task::Poll; use std::time::Duration; use uring_runtime::reactor::IoBuffer; +pub mod direct; pub mod listener; pub mod origin_directory; +pub use direct::DIRECT_WINDOW_PAGES; +use direct::DirectPage; +use direct::DirectSink; +use direct::GpuService; + use racer_object_wire::client::MAX_HEAD_BYTES; pub use racer_object_wire::client::ReadKind; use racer_object_wire::client::frame; @@ -115,6 +121,7 @@ async fn handle_read_result( scope: &RequestScope, observation: &mut crate::telemetry::RequestMetrics, timeout: Duration, + sink: Option>, ) -> Result> { let response = match read_result { Ok(response) => response, @@ -162,7 +169,19 @@ async fn handle_read_result( > 4; let mut watch = watch_room.then(|| responses.io.disconnected(&connection, &watch_scope)); let mut poll_disconnect = !watch_room; - let mut send = if matches!(kind, ReadKind::Subscription { .. }) { + let mut send = if let Some(sink) = sink { + responses.send_direct( + connection, + response, + kind, + sink, + scope, + observation, + timeout, + ) + } else if kind.direct().is_some() { + Box::pin(async { Err(Error::Internal) }) + } else if matches!(kind, ReadKind::Subscription { .. }) { responses.send_subscription(connection, response, scope, observation, timeout) } else { responses.send_observed(connection, response, scope, observation) @@ -225,6 +244,8 @@ pub struct Responses { io: Rc, delivery: Rc, + + gpu: Option>, } /// Duplex subscription framing. Delivered leases stay pinned until exact release. @@ -619,15 +640,190 @@ impl Responses { }) } + /// Deliver a direct read: write each page through `sink`, then send its + /// payload-less page frame. The sink is finished before the kind 2 frame, + /// and on a best-effort basis when delivery fails. + #[allow(clippy::too_many_arguments)] + pub(crate) fn send_direct<'a>( + &'a self, + connection: ConnectionLease, + response: ReadResponse, + kind: &'a ReadKind, + mut sink: Box, + scope: &'a RequestScope, + observation: &'a mut crate::telemetry::RequestMetrics, + timeout: Duration, + ) -> Operation<'a, ConnectionLease> { + Box::pin(async move { + let result = self + .send_direct_inner( + connection, + response, + kind, + sink.as_mut(), + scope, + observation, + timeout, + ) + .await; + if result.is_err() { + // Give device work a short, independent bound to settle. + if let Ok(final_scope) = RequestScope::new( + scope.request, + uring_runtime::environment::now() + Duration::from_secs(1), + ) { + let _ = sink.finish(&final_scope).await; + } + } + result + }) + } + + #[allow(clippy::too_many_arguments)] + async fn send_direct_inner( + &self, + mut connection: ConnectionLease, + mut response: ReadResponse, + kind: &ReadKind, + sink: &mut dyn DirectSink, + scope: &RequestScope, + observation: &mut crate::telemetry::RequestMetrics, + timeout: Duration, + ) -> Result { + let delivery = kind.direct().ok_or(Error::Internal)?; + let destination = kind.destination().ok_or(Error::Internal)?; + let head = racer_object_wire::client::direct_head( + &response.metadata, + response.payload.range(), + delivery, + )?; + let expected = response.payload.range().map_or(0, |r| r.len()); + if expected > destination.length { + return Err(Error::InvalidRequest); + } + let mut first = if let ReadPayload::Stream(stream) = &mut response.payload { + match stream.next_slice().await { + Ok(Some(reader)) => Some(reader), + result => { + let error = result.err().unwrap_or(Error::BadGateway); + self.observer.record( + crate::telemetry::Failure::new(crate::telemetry::Stage::FirstSlice, error) + .request(scope), + ); + observation.fail(error); + return self.send_error(connection, error, scope).await; + } + } + } else { + None + }; + connection.poison(); + connection = self.io.send_head(connection, head, scope).await?.connection; + let mut sent = 0; + let mut pages = 0; + if let ReadPayload::Stream(stream) = &mut response.payload { + let range = stream.range(); + stream.enable_progress(timeout); + while sent < expected { + let mut progress_scope = scope.clone(); + progress_scope.deadline.0 = uring_runtime::environment::now() + timeout; + let reader = match first.take() { + Some(reader) => reader, + None => std::future::poll_fn(|cx| { + progress_scope.check()?; + stream.next_slice().as_mut().poll(cx) + }) + .await + .inspect_err(|&error| { + self.observer.record( + crate::telemetry::Failure::new( + crate::telemetry::Stage::NextSlice, + error, + ) + .request(scope), + ); + })? + .ok_or(Error::BadGateway)?, + }; + let slice = reader.slice(); + if range.slice_at(slice.page)? != Some(slice) + || slice.length == 0 + || u64::from(slice.length) > expected - sent + { + return Err(Error::BadGateway); + } + let offset = slice + .page + .0 + .checked_mul(PAGE_BYTES) + .and_then(|n| n.checked_add(u64::from(slice.offset))) + .ok_or(Error::BadGateway)?; + // The parser and validate bound offset + range length. + let target = destination + .offset + .checked_add(offset - range.start()) + .ok_or(Error::BadGateway)?; + { + let mut write = + sink.deliver(DirectPage::Plaintext(&reader), target, &progress_scope); + std::future::poll_fn(|cx| { + if let Poll::Ready(result) = write.as_mut().poll(cx) { + return Poll::Ready(result); + } + // Keep the fetch window moving while the write runs. + stream.poll_prefetch(cx); + Poll::Pending + }) + .await?; + } + drop(reader); + // The page is written: no client credit gates its release. + stream.release_page(slice.page, slice.length)?; + connection = send_frame( + &self.io, + connection, + frame(1, slice.page.0, offset, slice.length), + &progress_scope, + ) + .await?; + sent += u64::from(slice.length); + pages += 1; + } + } + let mut final_scope = scope.clone(); + final_scope.deadline.0 = uring_runtime::environment::now() + timeout; + sink.finish(&final_scope).await?; + connection = + send_frame(&self.io, connection, frame(2, pages, sent, 0), &final_scope).await?; + observation.success(); + Ok(connection) + } + /// Attach delivery resources with a no-op observer until configured. pub fn new(io: Rc, delivery: Rc) -> Self { Self { io, delivery, observer: Observer::default(), + gpu: None, } } + /// Serve `gpu-read` with this backend. Without one, `gpu-read` is 501. + pub fn with_gpu(mut self, gpu: Rc) -> Self { + self.gpu = Some(gpu); + self + } + + /// Check the descriptors that came with a request and open its sink. + pub(crate) fn open_sink( + &self, + kind: &ReadKind, + received: http1::connection::ReceivedDescriptors, + ) -> Result>> { + direct::open_sink(kind, received, &self.delivery, self.gpu.as_ref()) + } + /// Route detailed delivery failures to the worker's observer. pub(crate) fn with_observer(mut self, observer: Observer) -> Self { self.observer = observer; @@ -686,6 +882,28 @@ impl Responses { subscription_head(&response.metadata, response.payload.range())?; return Ok(()); } + ReadKind::HostRead { + range, destination, .. + } + | ReadKind::GpuRead { + range, destination, .. + } => { + let expected = expected_subscription_range(*range, response.metadata.length)?; + if response.payload.range() != expected { + return Err(Error::BadGateway); + } + // Reject before the 200 head: the window must hold the range. + if expected.map_or(0, |range| range.len()) > destination.length { + return Err(Error::InvalidRequest); + } + let delivery = kind.direct().ok_or(Error::Internal)?; + racer_object_wire::client::direct_head( + &response.metadata, + response.payload.range(), + delivery, + )?; + return Ok(()); + } ReadKind::Head | ReadKind::HeadPinned { .. } => { if !matches!(&response.payload, ReadPayload::Empty) { return Err(Error::BadGateway); @@ -769,6 +987,7 @@ fn header(name: &str, value: impl AsRef<[u8]>) -> Header { fn error_head(error: Error) -> Result { let status = match error { Error::InvalidRequest | Error::InvalidRange => 400, + Error::NotImplemented => 501, Error::MethodNotAllowed => 405, Error::HeaderTooLarge => 431, Error::Unauthorized | Error::OriginRejected => 401, diff --git a/cmd/racer-dataplane/src/client/direct.rs b/cmd/racer-dataplane/src/client/direct.rs new file mode 100644 index 000000000..668443911 --- /dev/null +++ b/cmd/racer-dataplane/src/client/direct.rs @@ -0,0 +1,208 @@ +//! Direct reads write object plaintext into a client descriptor, not the socket. +//! +//! `host-read` writes into a regular file (normally a memfd) with positioned +//! io_uring writes. `gpu-read` hands each page to a [`GpuService`] backend. Both +//! share one request path through the [`DirectSink`] seam: the client code +//! resolves the range, gets each verified page from the Get read path, and asks +//! the sink to deliver it at a destination offset. The sink resolves the future +//! only when the bytes are visible to the client, so the next page frame can +//! go out. +//! +//! The client sends exactly one descriptor with the request head. Any other +//! count, a descriptor on another request, or truncated ancillary data is a 400. + +use crate::client::ReadKind; +use crate::error::Error; +use crate::error::Operation; +use crate::error::Result; +use crate::http::Delivery; +use crate::http::ReaderLease; +use crate::runtime::RequestScope; +use http1::connection::ReceivedDescriptors; +pub use racer_object_wire::client::Destination; +pub use racer_object_wire::client::DirectDelivery; +use std::os::fd::AsRawFd; +use std::os::fd::OwnedFd; +use std::rc::Rc; +use uring_runtime::reactor::descriptor::Descriptor; + +/// Pages the dataplane fetches ahead for one direct read. Clients send no +/// credits, so this window alone bounds the plaintext a direct read pins. +pub const DIRECT_WINDOW_PAGES: usize = 4; + +/// One page handed to a [`DirectSink`]. +/// +/// Today every page arrives as verified plaintext. A GPU backend that decrypts +/// on the device will add a variant that carries the page identity and asks the +/// sink to fetch ciphertext itself; the enum is non-exhaustive for that reason. +#[non_exhaustive] +pub enum DirectPage<'a> { + /// Verified plaintext already in memory. [`ReaderLease::slice_bytes`] holds + /// exactly the bytes to write; [`ReaderLease::page`] can be cloned to keep + /// the page alive past this call. + Plaintext(&'a ReaderLease), +} + +impl DirectPage<'_> { + /// Number of bytes this page writes. + pub fn len(&self) -> u64 { + match self { + Self::Plaintext(reader) => u64::from(reader.slice().length), + } + } + + /// Whether this page writes nothing. Pages are never empty in practice. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// Where a direct read writes. One sink serves one request and owns the +/// client's descriptor; dropping the sink must release it. +/// +/// The caller calls [`Self::deliver`] once per page, in page order, and waits +/// for each call before the next. It calls [`Self::finish`] once after the last +/// page, and on a best-effort basis after a failure, before the kind 2 frame. +/// A sink may be dropped without `finish`, for example when the client +/// disconnects; its `Drop` must then keep any resource that a device may still +/// write rather than free it. +pub trait DirectSink { + /// Write `page` at `destination_offset` in the client's destination. The + /// caller guarantees the bytes fall inside the request's destination + /// window. Resolve only when the bytes are visible to the client. + fn deliver<'a>( + &'a mut self, + page: DirectPage<'a>, + destination_offset: u64, + scope: &'a RequestScope, + ) -> Operation<'a, ()>; + + /// Wait for all work that touches the destination, then release it. + fn finish<'a>(&'a mut self, scope: &'a RequestScope) -> Operation<'a, ()>; +} + +/// A GPU backend for `gpu-read`. Without one, `gpu-read` answers 501. +pub trait GpuService { + /// Import the client's exported allocation and return a sink for one + /// request. Called after the request is validated and before any read. + /// Return [`Error::NotImplemented`] when the allocation is on a GPU the + /// dataplane cannot use; [`Error::InvalidRequest`] for a bad handle. + fn open( + &self, + descriptor: OwnedFd, + destination: Destination, + allocation_size: u64, + ) -> Result>; +} + +/// Writes pages into a client file with positioned io_uring writes. +pub(crate) struct HostSink { + file: Rc, + + delivery: Rc, + + window: Destination, +} + +impl HostSink { + /// Accept only a regular file opened for writing. Never map it, so a + /// client that truncates the file cannot fault the dataplane. + pub(crate) fn open( + descriptor: OwnedFd, + window: Destination, + delivery: Rc, + ) -> Result { + // SAFETY: fstat writes only into the zeroed stat buffer. + let mut stat: libc::stat = unsafe { std::mem::zeroed() }; + if unsafe { libc::fstat(descriptor.as_raw_fd(), &mut stat) } != 0 + || stat.st_mode & libc::S_IFMT != libc::S_IFREG + { + return Err(Error::InvalidRequest); + } + // SAFETY: F_GETFL only reads descriptor flags. + let flags = unsafe { libc::fcntl(descriptor.as_raw_fd(), libc::F_GETFL) }; + if flags < 0 || flags & libc::O_ACCMODE == libc::O_RDONLY || flags & libc::O_APPEND != 0 { + return Err(Error::InvalidRequest); + } + Ok(Self { + file: Rc::new(Descriptor::from(descriptor)), + delivery, + window, + }) + } +} + +impl DirectSink for HostSink { + fn deliver<'a>( + &'a mut self, + page: DirectPage<'a>, + destination_offset: u64, + scope: &'a RequestScope, + ) -> Operation<'a, ()> { + Box::pin(async move { + let end = destination_offset + .checked_add(page.len()) + .ok_or(Error::InvalidRange)?; + if destination_offset < self.window.offset || end > self.window.end() { + return Err(Error::InvalidRange); + } + match page { + DirectPage::Plaintext(reader) => { + self.delivery + .write_slice_at(reader, self.file.clone(), destination_offset, scope) + .await + } + } + }) + } + + fn finish<'a>(&'a mut self, _scope: &'a RequestScope) -> Operation<'a, ()> { + // Completed io_uring writes are already visible through the page cache. + Box::pin(async { Ok(()) }) + } +} + +/// Pair the descriptors received with a request head with its parsed kind, +/// and open the sink for a direct read. Unclaimed descriptors close on return. +pub(crate) fn open_sink( + kind: &ReadKind, + received: ReceivedDescriptors, + delivery: &Rc, + gpu: Option<&Rc>, +) -> Result>> { + let ReceivedDescriptors { + mut descriptors, + truncated, + } = received; + if truncated { + return Err(Error::InvalidRequest); + } + let Some(mode) = kind.direct() else { + return if descriptors.is_empty() { + Ok(None) + } else { + Err(Error::InvalidRequest) + }; + }; + let (Some(descriptor), true) = (descriptors.pop(), descriptors.is_empty()) else { + return Err(Error::InvalidRequest); + }; + let destination = kind.destination().ok_or(Error::Internal)?; + match (mode, kind) { + (DirectDelivery::Host, _) => Ok(Some(Box::new(HostSink::open( + descriptor, + destination, + delivery.clone(), + )?))), + ( + DirectDelivery::Gpu, + ReadKind::GpuRead { + allocation_size, .. + }, + ) => gpu + .ok_or(Error::NotImplemented)? + .open(descriptor, destination, *allocation_size) + .map(Some), + _ => Err(Error::Internal), + } +} diff --git a/cmd/racer-dataplane/src/client/listener.rs b/cmd/racer-dataplane/src/client/listener.rs index 8aaaf077f..d4af594ea 100644 --- a/cmd/racer-dataplane/src/client/listener.rs +++ b/cmd/racer-dataplane/src/client/listener.rs @@ -948,8 +948,9 @@ async fn serve_connection( )?; let idle_scope = new_scope(timeout, cancellation.clone())?; deadline.set(idle_scope.deadline.0); + // Read with recvmsg so a direct read's descriptor arrives with its head. let received = io - .receive_request_head_limited(connection, &idle_scope, parser.header_limit()) + .receive_request_head_with_descriptors(connection, &idle_scope, parser.header_limit()) .await?; connection = received.connection; if !retirement_request && retired.load(std::sync::atomic::Ordering::Acquire) { @@ -962,8 +963,14 @@ async fn serve_connection( let scope = new_scope(timeout, cancellation.clone())?; deadline.set(scope.deadline.0); let scope = &scope; - let request = match received.value.and_then(|head| parser.parse(cache, head)) { - Ok(request) => request, + // Every descriptor must be claimed by a direct read; the rest close here. + let parsed = received.value.and_then(|received| { + let request = parser.parse(cache, received.head)?; + let sink = responses.open_sink(&request.kind, received.descriptors)?; + Ok((request, sink)) + }); + let (request, sink) = match parsed { + Ok(parsed) => parsed, Err(error) => { observation.fail(error); responses.send_error(connection, error, scope).await?; @@ -1015,6 +1022,7 @@ async fn serve_connection( scope, &mut observation, timeout, + sink, ) .await? { diff --git a/cmd/racer-dataplane/src/client/tests.rs b/cmd/racer-dataplane/src/client/tests.rs index c9d00c71a..506ae718b 100644 --- a/cmd/racer-dataplane/src/client/tests.rs +++ b/cmd/racer-dataplane/src/client/tests.rs @@ -1095,6 +1095,7 @@ impl Fixture { &scope, &mut observation, Duration::from_secs(5), + None, ) .await?; Ok(()) diff --git a/cmd/racer-dataplane/src/error.rs b/cmd/racer-dataplane/src/error.rs index c16901a5a..05be6fc6c 100644 --- a/cmd/racer-dataplane/src/error.rs +++ b/cmd/racer-dataplane/src/error.rs @@ -67,6 +67,8 @@ boundary_errors! { CorruptRecord, MissingKey, DirectIoUnsupported, + /// The request is valid, but this dataplane cannot serve it (HTTP 501). + NotImplemented, Io, Os(errno: i32), } diff --git a/cmd/racer-dataplane/src/http.rs b/cmd/racer-dataplane/src/http.rs index 001b39585..ef536f28c 100644 --- a/cmd/racer-dataplane/src/http.rs +++ b/cmd/racer-dataplane/src/http.rs @@ -500,6 +500,15 @@ impl ReaderLease { pub fn remaining(&self) -> usize { self.slice.length as usize - self.sent } + /// The verified plaintext of this slice only. + pub fn slice_bytes(&self) -> &[u8] { + let start = self.slice.offset as usize; + &self.page.bytes()[start..start + self.slice.length as usize] + } + /// The verified page this slice pins. + pub fn page(&self) -> &VerifiedPage { + &self.page + } } struct DeliveryOwner(ReaderLease); impl http1::delivery::DirectOwner for DeliveryOwner { @@ -656,6 +665,40 @@ impl Delivery { }) } } +impl Delivery { + /// Write the whole slice into `file` starting at `offset`, with positioned + /// io_uring writes from the shared page. Short writes continue where they + /// stopped; a zero-byte write is an I/O error. Never maps the file. + pub(crate) fn write_slice_at<'a>( + &'a self, + reader: &ReaderLease, + file: Rc, + offset: u64, + scope: &'a RequestScope, + ) -> Operation<'a, ()> { + let page = reader.page.clone(); + let start = reader.slice.offset as usize; + let length = reader.slice.length as usize; + Box::pin(async move { + let mut written = 0; + while written < length { + let view = PageSendRange::new(page.clone(), start + written..start + length)?; + let position = offset + .checked_add(written as u64) + .ok_or(Error::InvalidRange)?; + let completed = self + .reactor + .write_shared_at(file.clone(), position, view, (), scope) + .await?; + if completed.bytes == 0 || completed.bytes > length - written { + return Err(Error::Io); + } + written += completed.bytes; + } + Ok(()) + }) + } +} fn validate_slice(page: &VerifiedPage, slice: PageSlice) -> Result<()> { let end = (slice.offset as usize) .checked_add(slice.length as usize) diff --git a/cmd/racer-dataplane/src/read.rs b/cmd/racer-dataplane/src/read.rs index e1635d647..7285674ed 100644 --- a/cmd/racer-dataplane/src/read.rs +++ b/cmd/racer-dataplane/src/read.rs @@ -16,6 +16,7 @@ use crate::model::ByteRange; use crate::model::MetadataSelector; use crate::model::ObjectId; use crate::model::ObjectMetadata; +use crate::model::PAGE_BYTES; use crate::model::ResolvedRange; use crate::peer::forwarding::VerifiedRequest; use crate::peer::protocol::FetchMode; @@ -169,7 +170,9 @@ impl Coordinator { retain, } = request; let selector = match &kind { - ReadKind::Subscription { pin, .. } => pin + ReadKind::Subscription { pin, .. } + | ReadKind::HostRead { pin, .. } + | ReadKind::GpuRead { pin, .. } => pin .clone() .map_or(MetadataSelector::Fresh, MetadataSelector::Pinned), ReadKind::Head => MetadataSelector::Fresh, @@ -185,6 +188,19 @@ impl Coordinator { ) .await?; validate_metadata(&metadata, &origin.object, &selector)?; + // Direct reads have no client credits. The dataplane paces them + // with a fixed window and releases each page after writing it. + let kind = match kind { + ReadKind::HostRead { range, .. } | ReadKind::GpuRead { range, .. } => { + ReadKind::Subscription { + pin: None, + range, + page_credits: crate::client::DIRECT_WINDOW_PAGES, + byte_credits: crate::client::DIRECT_WINDOW_PAGES as u64 * PAGE_BYTES, + } + } + kind => kind, + }; match kind { ReadKind::Subscription { range, @@ -244,6 +260,8 @@ impl Coordinator { retain_until, }) } + // Mapped to Subscription above. + ReadKind::HostRead { .. } | ReadKind::GpuRead { .. } => Err(Error::Internal), } }) } From 9c566bc975328c4c16ad9c96529d2e4d9027b6ac Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 19:19:57 +0000 Subject: [PATCH 24/87] test(racer): cover host-read and gpu-read over the client socket --- cmd/racer-dataplane/src/client/tests.rs | 1 + .../src/client/tests/direct.rs | 500 ++++++++++++++++++ docs/content/reference/racer.md | 33 ++ 3 files changed, 534 insertions(+) create mode 100644 cmd/racer-dataplane/src/client/tests/direct.rs diff --git a/cmd/racer-dataplane/src/client/tests.rs b/cmd/racer-dataplane/src/client/tests.rs index 506ae718b..3d323b103 100644 --- a/cmd/racer-dataplane/src/client/tests.rs +++ b/cmd/racer-dataplane/src/client/tests.rs @@ -4074,3 +4074,4 @@ fn handoff_acceptor_drains_backlog_and_wakes_only_while_one_remains() { } assert_eq!(worker.listeners.active_connections_for(&definition().id), 3); } +mod direct; diff --git a/cmd/racer-dataplane/src/client/tests/direct.rs b/cmd/racer-dataplane/src/client/tests/direct.rs new file mode 100644 index 000000000..01016f0dc --- /dev/null +++ b/cmd/racer-dataplane/src/client/tests/direct.rs @@ -0,0 +1,500 @@ +//! `host-read` and `gpu-read` over the real client socket with a fake origin. +use super::*; +use crate::client::direct::Destination; +use crate::client::direct::DirectPage; +use crate::client::direct::DirectSink; +use crate::client::direct::GpuService; +use crate::model::CacheKey; +use crate::model::ObjectId; +use crate::model::PAGE_BYTES; +use std::io::Seek; +use std::os::fd::FromRawFd; +use std::os::fd::OwnedFd; +use std::os::fd::RawFd; + +static NAMES: AtomicUsize = AtomicUsize::new(0); + +/// A memfd with a process-unique name, so open copies can be counted. +struct Memfd { + file: File, + name: String, +} + +impl Memfd { + fn new() -> Self { + let name = format!( + "racer-direct-{}-{}", + std::process::id(), + NAMES.fetch_add(1, Ordering::Relaxed) + ); + let c_name = CString::new(name.clone()).unwrap(); + let fd = unsafe { libc::memfd_create(c_name.as_ptr(), libc::MFD_CLOEXEC) }; + assert!(fd >= 0); + Self { + file: unsafe { File::from_raw_fd(fd) }, + name, + } + } + + /// Descriptors in this process that refer to this memfd, including ours. + fn open_copies(&self) -> usize { + let exact = format!("/memfd:{} (deleted)", self.name); + fs::read_dir("/proc/self/fd") + .unwrap() + .filter_map(|entry| fs::read_link(entry.ok()?.path()).ok()) + .filter(|target| target.to_string_lossy() == exact) + .count() + } + + fn contents(&mut self) -> Vec { + let mut output = Vec::new(); + self.file.seek(std::io::SeekFrom::Start(0)).unwrap(); + self.file.read_to_end(&mut output).unwrap(); + output + } +} + +/// Send `bytes` with `fds` attached as one `SCM_RIGHTS` message. +fn send_with_fds(socket: &UnixStream, bytes: &[u8], fds: &[RawFd]) { + let payload = std::mem::size_of_val(fds); + let space = unsafe { libc::CMSG_SPACE(payload as u32) } as usize; + let mut control = vec![0u64; space.div_ceil(8)]; + let mut iov = libc::iovec { + iov_base: bytes.as_ptr().cast_mut().cast(), + iov_len: bytes.len(), + }; + let mut header: libc::msghdr = unsafe { std::mem::zeroed() }; + header.msg_iov = &mut iov; + header.msg_iovlen = 1; + if !fds.is_empty() { + header.msg_control = control.as_mut_ptr().cast(); + header.msg_controllen = space as _; + unsafe { + let cmsg = libc::CMSG_FIRSTHDR(&header); + (*cmsg).cmsg_level = libc::SOL_SOCKET; + (*cmsg).cmsg_type = libc::SCM_RIGHTS; + (*cmsg).cmsg_len = libc::CMSG_LEN(payload as u32) as _; + std::ptr::copy_nonoverlapping( + fds.as_ptr().cast::(), + libc::CMSG_DATA(cmsg), + payload, + ); + } + } + let sent = unsafe { libc::sendmsg(socket.as_raw_fd(), &header, 0) }; + assert_eq!(sent, bytes.len() as isize); +} + +fn direct_request(suffix: &str, fields: &str) -> Vec { + format!( + "POST /v1/objects/{}/{suffix} HTTP/1.1\r\nHost: racer\r\nContent-Length: 0\r\n{fields}\r\n", + "0".repeat(64) + ) + .into_bytes() +} + +/// A fixture whose fake origin serves `body` as version "v1". +fn direct_fixture(body: Vec, gpu: Option>) -> Fixture { + let mut fixture = Fixture::new(); + let delivery = Rc::new(Delivery::new( + fixture.reactor.clone(), + Duration::from_secs(2), + )); + let mut responses = Responses::new(fixture.listeners.io.clone(), delivery.clone()); + if let Some(gpu) = gpu { + responses = responses.with_gpu(gpu); + } + fixture.listeners.responses = Rc::new(responses); + let worker = ReadWorker::new( + definition(), + ObjectMetadata { + content_type: None, + version: ObjectVersion { + object: ObjectId { + cache: definition().id, + key: CacheKey([0; 32]), + }, + etag: StrongEtag::parse(b"\"v1\"").unwrap(), + }, + length: body.len() as u64, + expires_at: ExpiresAt::from_system_time(UNIX_EPOCH + Duration::from_millis(1234)) + .unwrap(), + }, + fixture.listeners.admission.clone(), + fixture.reactor.clone(), + delivery, + 1, + ); + worker.origin.set_body(body); + fixture.listeners.reads = worker.coordinator.clone(); + fixture.worker = Some(worker); + fixture.reconcile(&[definition()]).unwrap(); + fixture +} + +/// Split a complete response into its lowercased head and body bytes. +fn split(output: &[u8]) -> (String, &[u8]) { + let end = output + .windows(4) + .position(|part| part == b"\r\n\r\n") + .expect("response head") + + 4; + ( + std::str::from_utf8(&output[..end]) + .unwrap() + .to_ascii_lowercase(), + &output[end..], + ) +} + +/// Decode 21-byte frames: (kind, number, offset, length). +fn frames(body: &[u8]) -> Vec<(u8, u64, u64, u32)> { + assert_eq!(body.len() % 21, 0, "body is whole frames"); + body.chunks(21) + .map(|frame| { + ( + frame[0], + u64::from_be_bytes(frame[1..9].try_into().unwrap()), + u64::from_be_bytes(frame[9..17].try_into().unwrap()), + u32::from_be_bytes(frame[17..].try_into().unwrap()), + ) + }) + .collect() +} + +/// Send one request with `fds` and read the response to EOF. +fn exchange(fixture: &Fixture, request: &[u8], fds: &[RawFd]) -> Vec { + let mut socket = fixture.connect(); + send_with_fds(&socket, request, fds); + fixture.receive(&mut socket, true) +} + +/// Pump until the connection and its descriptors are gone. +fn settle(fixture: &Fixture) { + for _ in 0..64 { + fixture.pump(16); + } + assert_eq!(fixture.listeners.active_connections(), 0); +} + +#[test] +fn host_read_writes_the_range_at_the_destination_offset() { + let fixture = direct_fixture(b"hello".to_vec(), None); + let mut memfd = Memfd::new(); + memfd.file.write_all(&[b'.'; 16]).unwrap(); + let output = exchange( + &fixture, + &direct_request( + "host-read", + "Range: bytes=1-3\r\nIf-Match: \"v1\"\r\nRacer-Destination-Offset: 10\r\n\ + Racer-Destination-Length: 3\r\nRacer-Page-Credits: 1\r\n", + ), + &[memfd.file.as_raw_fd()], + ); + let (head, body) = split(&output); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert!(head.contains("racer-delivery: host\r\n"), "{head}"); + assert!(head.contains("content-length: 42\r\n"), "{head}"); + assert!(head.contains("racer-range-start: 1\r\n"), "{head}"); + assert!(head.contains("racer-range-end: 4\r\n"), "{head}"); + assert!(head.contains("connection: close\r\n"), "{head}"); + assert_eq!(frames(body), [(1, 0, 1, 3), (2, 1, 3, 0)]); + assert_eq!(memfd.contents(), b"..........ell..."); + settle(&fixture); + assert_eq!(memfd.open_copies(), 1, "dataplane closed its copy"); + assert_no_body_leases(&fixture); +} + +#[test] +fn host_read_spans_pages_and_whole_objects() { + let length = PAGE_BYTES as usize + 3; + let body: Vec = (0..length).map(|i| (i % 251) as u8).collect(); + let fixture = direct_fixture(body.clone(), None); + let mut memfd = Memfd::new(); + let output = exchange( + &fixture, + &direct_request( + "host-read", + &format!("Racer-Destination-Offset: 0\r\nRacer-Destination-Length: {length}\r\n"), + ), + &[memfd.file.as_raw_fd()], + ); + let (head, body_frames) = split(&output); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert!(head.contains("content-length: 63\r\n"), "{head}"); + assert_eq!( + frames(body_frames), + [ + (1, 0, 0, PAGE_BYTES as u32), + (1, 1, PAGE_BYTES, 3), + (2, 2, length as u64, 0) + ] + ); + assert!(memfd.contents() == body, "destination matches the object"); + settle(&fixture); + assert_eq!(memfd.open_copies(), 1); +} + +#[test] +fn host_read_descriptor_errors_are_400_and_close_descriptors() { + let fields = "Racer-Destination-Offset: 0\r\nRacer-Destination-Length: 5\r\n"; + let fixture = direct_fixture(b"hello".to_vec(), None); + let memfd = Memfd::new(); + let fd = memfd.file.as_raw_fd(); + let plain = request("POST", ""); + for (request, fds) in [ + // Missing descriptor. + (direct_request("host-read", fields), vec![]), + // Two descriptors. + (direct_request("host-read", fields), vec![fd, fd]), + // A descriptor on a plain Get. + (plain.clone(), vec![fd]), + // A descriptor on a HEAD. + (request("HEAD", ""), vec![fd]), + // Missing destination field. + ( + direct_request("host-read", "Racer-Destination-Offset: 0\r\n"), + vec![fd], + ), + ] { + let output = exchange(&fixture, &request, &fds); + let (head, body) = split(&output); + assert!( + head.starts_with("http/1.1 400"), + "{head} {:?}", + String::from_utf8_lossy(&request) + ); + assert!(body.is_empty()); + settle(&fixture); + assert_eq!( + memfd.open_copies(), + 1, + "{}", + String::from_utf8_lossy(&request) + ); + } + // Plain requests without descriptors still work. + let output = exchange(&fixture, &plain, &[]); + assert!(split(&output).0.starts_with("http/1.1 200")); +} + +#[test] +fn host_read_rejects_non_regular_and_read_only_descriptors() { + let fields = "Racer-Destination-Offset: 0\r\nRacer-Destination-Length: 5\r\n"; + let fixture = direct_fixture(b"hello".to_vec(), None); + let mut fds = [0; 2]; + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + let mut read = unsafe { File::from_raw_fd(fds[0]) }; + let write = unsafe { OwnedFd::from_raw_fd(fds[1]) }; + let output = exchange( + &fixture, + &direct_request("host-read", fields), + &[write.as_raw_fd()], + ); + assert!(split(&output).0.starts_with("http/1.1 400")); + settle(&fixture); + drop(write); + let mut byte = [0]; + assert_eq!(read.read(&mut byte).unwrap(), 0, "pipe write end leaked"); + + let memfd = Memfd::new(); + let path = format!("/proc/self/fd/{}", memfd.file.as_raw_fd()); + let read_only = File::open(path).unwrap(); + let output = exchange( + &fixture, + &direct_request("host-read", fields), + &[read_only.as_raw_fd()], + ); + assert!(split(&output).0.starts_with("http/1.1 400")); + settle(&fixture); + drop(read_only); + assert_eq!(memfd.open_copies(), 1); +} + +#[test] +fn host_read_range_and_version_errors_keep_get_statuses() { + let fixture = direct_fixture(b"hello".to_vec(), None); + let mut memfd = Memfd::new(); + for (fields, status) in [ + // Destination smaller than the range. + ( + "Range: bytes=0-4\r\nRacer-Destination-Offset: 0\r\nRacer-Destination-Length: 4\r\n", + "400", + ), + ( + "If-Match: \"v2\"\r\nRacer-Destination-Offset: 0\r\nRacer-Destination-Length: 5\r\n", + "412", + ), + ( + "Range: bytes=9-\r\nRacer-Destination-Offset: 0\r\nRacer-Destination-Length: 5\r\n", + "416", + ), + ] { + let output = exchange( + &fixture, + &direct_request("host-read", fields), + &[memfd.file.as_raw_fd()], + ); + let (head, _) = split(&output); + assert!(head.starts_with(&format!("http/1.1 {status}")), "{head}"); + settle(&fixture); + assert_eq!(memfd.open_copies(), 1, "{fields}"); + } + assert!(memfd.contents().is_empty(), "nothing was written"); + assert_no_body_leases(&fixture); +} + +#[test] +fn host_read_client_disconnect_closes_descriptor_and_releases_pages() { + let length = 2 * PAGE_BYTES as usize + 1; + let fixture = direct_fixture(vec![b'x'; length], None); + let memfd = Memfd::new(); + let mut socket = fixture.connect(); + send_with_fds( + &socket, + &direct_request( + "host-read", + &format!("Racer-Destination-Offset: 0\r\nRacer-Destination-Length: {length}\r\n"), + ), + &[memfd.file.as_raw_fd()], + ); + let output = fixture.receive(&mut socket, false); + assert!(split(&output).0.starts_with("http/1.1 200")); + drop(socket); + settle(&fixture); + assert_eq!(memfd.open_copies(), 1); + fixture.listeners.admission.reclaim_buffers(); + assert_no_body_leases(&fixture); +} + +#[test] +fn gpu_read_is_validated_then_501_without_a_backend() { + let fixture = direct_fixture(b"hello".to_vec(), None); + let memfd = Memfd::new(); + let fd = memfd.file.as_raw_fd(); + for (fields, fds, status) in [ + ( + "Racer-Destination-Offset: 0\r\nRacer-Destination-Length: 5\r\nRacer-Allocation-Size: 5\r\n", + vec![fd], + "501", + ), + // Validation precedes the 501. + ( + "Racer-Destination-Offset: 1\r\nRacer-Destination-Length: 5\r\nRacer-Allocation-Size: 5\r\n", + vec![fd], + "400", + ), + ( + "Racer-Destination-Offset: 0\r\nRacer-Destination-Length: 5\r\nRacer-Allocation-Size: 5\r\n", + vec![], + "400", + ), + ] { + let output = exchange(&fixture, &direct_request("gpu-read", fields), &fds); + let (head, body) = split(&output); + assert!(head.starts_with(&format!("http/1.1 {status}")), "{head}"); + assert!(body.is_empty()); + settle(&fixture); + assert_eq!(memfd.open_copies(), 1); + } +} + +/// Records what a GPU backend would be asked to do. +#[derive(Default)] +struct Recorded { + opened: Vec<(Destination, u64)>, + writes: Vec<(u64, Vec)>, + finished: usize, +} + +struct FakeGpu(Rc>); + +struct FakeSink(Rc>, #[allow(dead_code)] OwnedFd); + +impl GpuService for FakeGpu { + fn open( + &self, + descriptor: OwnedFd, + destination: Destination, + allocation_size: u64, + ) -> Result> { + self.0 + .borrow_mut() + .opened + .push((destination, allocation_size)); + Ok(Box::new(FakeSink(self.0.clone(), descriptor))) + } +} + +impl DirectSink for FakeSink { + fn deliver<'a>( + &'a mut self, + page: DirectPage<'a>, + destination_offset: u64, + _scope: &'a RequestScope, + ) -> crate::error::Operation<'a, ()> { + let DirectPage::Plaintext(reader) = page; + let bytes = reader.slice_bytes().to_vec(); + let recorded = self.0.clone(); + Box::pin(async move { + // Complete on a later poll, as a helper thread would. + let mut ready = false; + std::future::poll_fn(|cx| { + if std::mem::replace(&mut ready, true) { + Poll::Ready(()) + } else { + cx.waker().wake_by_ref(); + Poll::Pending + } + }) + .await; + recorded + .borrow_mut() + .writes + .push((destination_offset, bytes)); + Ok(()) + }) + } + + fn finish<'a>(&'a mut self, _scope: &'a RequestScope) -> crate::error::Operation<'a, ()> { + self.0.borrow_mut().finished += 1; + Box::pin(async { Ok(()) }) + } +} + +#[test] +fn gpu_read_delivers_pages_through_the_backend_seam() { + let recorded = Rc::new(RefCell::new(Recorded::default())); + let fixture = direct_fixture(b"hello".to_vec(), Some(Rc::new(FakeGpu(recorded.clone())))); + let memfd = Memfd::new(); + let output = exchange( + &fixture, + &direct_request( + "gpu-read", + "Range: bytes=2-\r\nRacer-Destination-Offset: 100\r\n\ + Racer-Destination-Length: 3\r\nRacer-Allocation-Size: 4096\r\n", + ), + &[memfd.file.as_raw_fd()], + ); + let (head, body) = split(&output); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert!(head.contains("racer-delivery: gpu\r\n"), "{head}"); + assert_eq!(frames(body), [(1, 0, 2, 3), (2, 1, 3, 0)]); + { + let recorded = recorded.borrow(); + assert_eq!( + recorded.opened, + [( + Destination { + offset: 100, + length: 3 + }, + 4096 + )] + ); + assert_eq!(recorded.writes, [(100, b"llo".to_vec())]); + assert_eq!(recorded.finished, 1); + } + settle(&fixture); + assert_eq!(memfd.open_copies(), 1); +} diff --git a/docs/content/reference/racer.md b/docs/content/reference/racer.md index 6ee86e924..2cc66f9eb 100644 --- a/docs/content/reference/racer.md +++ b/docs/content/reference/racer.md @@ -354,6 +354,39 @@ Older dataplanes send pages out of order unless the client sends before SDKs. Older peers accept only one page request at a time from a node. The dataplane detects this and falls back to one request at a time for that peer. +### Host reads + +A client on the node can ask the dataplane to write an object straight into a +file it owns, such as a `memfd`, instead of streaming it over the socket: + +``` +POST /v1/objects//host-read HTTP/1.1 +Host: racer +Content-Length: 0 +Range: bytes=- optional +If-Match: "" optional +Racer-Destination-Offset: +Racer-Destination-Length: +``` + +- Send exactly one file descriptor with the request, as `SCM_RIGHTS` data on + the same `sendmsg` as the start of the request. It must be a regular file + opened for writing, without `O_APPEND`. +- The bytes at object offset `o` go to file offset + `Racer-Destination-Offset + (o - range start)`. +- The dataplane answers 400 if there is no descriptor, more than one, a + descriptor on any other request, a file that is not regular, or a length + smaller than the range. Range and version errors keep their usual statuses. +- A success has `Racer-Delivery: host`. The body uses the normal frames, but + page frames carry no data: a page frame means that page is already in the + file. The last frame is the usual completion frame. The client sends no + credits. +- If the read fails partway, the connection closes without the completion frame + and the file may hold partial data. + +`POST /v1/objects//gpu-read` is reserved for GPU memory. This dataplane +answers it with 501. + ### Warm and hard retention Send `HEAD` on the client socket with `Cache-Control: max-age=` to request From 911dadea205c2112c626f5c1acc631b34aa70176 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 19:36:38 +0000 Subject: [PATCH 25/87] test(racer): wait for direct-read connections to close on a deadline --- cmd/racer-dataplane/src/client/tests/direct.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/cmd/racer-dataplane/src/client/tests/direct.rs b/cmd/racer-dataplane/src/client/tests/direct.rs index 01016f0dc..bafbbf25b 100644 --- a/cmd/racer-dataplane/src/client/tests/direct.rs +++ b/cmd/racer-dataplane/src/client/tests/direct.rs @@ -169,12 +169,21 @@ fn exchange(fixture: &Fixture, request: &[u8], fds: &[RawFd]) -> Vec { fixture.receive(&mut socket, true) } -/// Pump until the connection and its descriptors are gone. +/// Pump until the connection and its descriptors are gone. Completions can +/// arrive late on a loaded host, so wait on a deadline, not a pump count. fn settle(fixture: &Fixture) { - for _ in 0..64 { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + fixture.pump(16); + if fixture.listeners.active_connections() == 0 { + break; + } + assert!(Instant::now() < deadline, "connection did not close"); + std::thread::yield_now(); + } + for _ in 0..8 { fixture.pump(16); } - assert_eq!(fixture.listeners.active_connections(), 0); } #[test] From 881384a47d76ae2be98928dcc06b6c1f65df39b6 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 19:54:08 +0000 Subject: [PATCH 26/87] fix(racer): do not mark host-read unsupported when the destination is too small --- clients/python/racer/src/racer/client.py | 11 +++++++---- clients/python/racer/tests/test_client.py | 5 ++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/clients/python/racer/src/racer/client.py b/clients/python/racer/src/racer/client.py index e541d2743..0be742e4c 100644 --- a/clients/python/racer/src/racer/client.py +++ b/clients/python/racer/src/racer/client.py @@ -352,15 +352,18 @@ def _stream( conn.send_head(self._head_bytes(head)) h = wire.parse_response_head(conn.read_head()) wire.check_error_status(h) - if on_head: - on_head() meta, first, end = self._check_selection(h, offset, length, if_match) pages = wire.page_count(first, end) body = wire.decimal(h.get("content-length")) if body != end - first + wire.FRAME_SIZE * (pages + 1): raise ProtocolError("bad Content-Length") if max_length is not None and end - first > max_length: + # An earlier direct mode may have refused this read with 400 + # for the same reason; that refusal says nothing about support + # (designs/racer-gpudirect.md, Fallback), so do not confirm it. raise RacerError("destination is smaller than the range") + if on_head: + on_head() seq = wire.Sequence(first, end) for _ in range(pages): f = wire.decode_frame(conn.read_exact(wire.FRAME_SIZE)) @@ -426,13 +429,13 @@ def _rpc( conn.send_head(self._head_bytes(head), fd) h = wire.parse_response_head(conn.read_head()) wire.check_error_status(h) - if on_head: - on_head() if h.get("racer-delivery") != mode: raise ProtocolError("bad Racer-Delivery") meta, first, end = self._check_selection(h, offset, length, if_match) if end - first > dest_length: raise ProtocolError("range is larger than the destination") + if on_head: + on_head() pages = wire.page_count(first, end) if wire.decimal(h.get("content-length")) != wire.FRAME_SIZE * (pages + 1): raise ProtocolError("bad Content-Length") diff --git a/clients/python/racer/tests/test_client.py b/clients/python/racer/tests/test_client.py index 0098f827a..48fccd303 100644 --- a/clients/python/racer/tests/test_client.py +++ b/clients/python/racer/tests/test_client.py @@ -175,8 +175,11 @@ def test_destination_too_small(server): with racer.HostBuffer(100) as buf: # host-read answers 400, which looks like an old dataplane, so the # stream fallback reports the size mismatch. + c = client(server) with pytest.raises(racer.RacerError, match="smaller"): - client(server).read_into_host(req, buf) + c.read_into_host(req, buf) + # The refusal was about this read, not about host-read support. + assert c.mode_state(racer.HOST) is None out = bytearray(100) with pytest.raises(racer.RacerError): client(server).read_into_host(req, out) From 1a2be9740eb07a0549641e2d796b251d232b941d Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 19:54:08 +0000 Subject: [PATCH 27/87] test(racer): align fake server with dataplane and add golden wire tests --- clients/python/racer/tests/fakeserver.py | 28 ++++- .../python/racer/tests/test_wire_golden.py | 119 ++++++++++++++++++ 2 files changed, 141 insertions(+), 6 deletions(-) create mode 100644 clients/python/racer/tests/test_wire_golden.py diff --git a/clients/python/racer/tests/fakeserver.py b/clients/python/racer/tests/fakeserver.py index ec4b3dc4a..bbe3d0a38 100644 --- a/clients/python/racer/tests/fakeserver.py +++ b/clients/python/racer/tests/fakeserver.py @@ -5,12 +5,14 @@ from __future__ import annotations import array +import fcntl import os import socket import stat import threading from dataclasses import dataclass, field +import racer from racer import _wire as wire @@ -102,6 +104,8 @@ def _send(conn, status, headers, body=b""): def _error(self, conn, status, size=None): h = [("Content-Length", "0")] + if status == 405: + h.append(("Allow", "HEAD, POST")) if status == 416: h.append(("Content-Range", f"bytes */{size}")) self._send(conn, status, h) @@ -138,8 +142,22 @@ def _dispatch(self, conn, method, target, headers, fds, seen, rest): key, _, mode = tail.partition("/") if mode and (self.old or mode not in ("host-read", "gpu-read")): return self._error(conn, 400) + # Order follows the dataplane: RequestParser::parse + # (object-wire/src/client.rs), then direct::open_sink and + # HostSink::open (src/client/direct.rs), then the object read. if mode and method != "POST": return self._error(conn, 405) + if mode: + if headers.get("content-length") != "0": + return self._error(conn, 400) + try: + off = wire.decimal(headers.get("racer-destination-offset")) + length = wire.decimal(headers.get("racer-destination-length")) + alloc = wire.decimal(headers.get("racer-allocation-size")) if mode == "gpu-read" else None + except racer.RacerError: + return self._error(conn, 400) + if length == 0 or off + length > wire.MAX_WIRE_INTEGER or (alloc is not None and off + length > alloc): + return self._error(conn, 400) if fds and not mode: return self._error(conn, 400) if mode and len(fds) != 1: @@ -148,8 +166,10 @@ def _dispatch(self, conn, method, target, headers, fds, seen, rest): return self._error(conn, self.gpu_status) if mode == "host-read" and self.host_status: return self._error(conn, self.host_status) - if mode == "host-read" and seen.fd_kind != "regular": - return self._error(conn, 400) + if mode == "host-read": + flags = fcntl.fcntl(fds[0], fcntl.F_GETFL) + if seen.fd_kind != "regular" or flags & os.O_ACCMODE == os.O_RDONLY or flags & os.O_APPEND: + return self._error(conn, 400) obj = self.objects.get(key) if obj is None: return self._error(conn, 404) @@ -182,12 +202,8 @@ def _dispatch(self, conn, method, target, headers, fds, seen, rest): ("Connection", "close"), ] if mode: - off = int(headers["racer-destination-offset"]) - length = int(headers["racer-destination-length"]) if length < end - first: return self._error(conn, 400) - if mode == "gpu-read" and off + length > int(headers["racer-allocation-size"]): - return self._error(conn, 400) self._send( conn, 200, diff --git a/clients/python/racer/tests/test_wire_golden.py b/clients/python/racer/tests/test_wire_golden.py new file mode 100644 index 000000000..4dd2e103f --- /dev/null +++ b/clients/python/racer/tests/test_wire_golden.py @@ -0,0 +1,119 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Golden bytes shared with the dataplane. + +Sources: cmd/racer-dataplane/object-wire/src/client.rs (RequestParser::parse, +direct_head, frame) and object-wire/tests/direct.rs. +""" + +import array +import fcntl +import os +import socket + +import pytest + +import racer +from racer import _wire as wire + +KEY = "ab" * 32 + + +def test_frame_layout_matches_rust_frame(): + # frame(kind, number, offset, length): u8, u64 BE, u64 BE, u32 BE. + raw = wire.encode_frame(1, 0x0102030405060708, 0x1112131415161718, 0x21222324) + assert raw == bytes([1, 1, 2, 3, 4, 5, 6, 7, 8, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x21, 0x22, 0x23, 0x24]) + # Direct completion is frame(2, pages, sent, 0). + assert wire.decode_frame(wire.encode_frame(2, 3, 2 * wire.PAGE_SIZE + 5, 0)) == wire.Frame(2, 3, 2 * wire.PAGE_SIZE + 5, 0) + + +def _capture(mode, call): + """Run one RPC against a socketpair-like listener and return head bytes and fds.""" + import tempfile + import threading + + d = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) + path = os.path.join(d, "s") + lst = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + lst.bind(path) + lst.listen(1) + got = {} + + def serve(): + conn, _ = lst.accept() + # The first recvmsg must carry the descriptor with the first head bytes. + data, anc, _, _ = conn.recvmsg(65536, socket.CMSG_SPACE(4 * 8)) + fds = array.array("i") + for level, typ, cdata in anc: + if level == socket.SOL_SOCKET and typ == socket.SCM_RIGHTS: + fds.frombytes(cdata[: len(cdata) - len(cdata) % 4]) + while b"\r\n\r\n" not in data: + data += conn.recv(65536) + got["head"] = data + got["flags"] = [fcntl.fcntl(fd, fcntl.F_GETFL) for fd in fds] + for fd in fds: + os.close(fd) + conn.sendall(b"HTTP/1.1 501 X\r\nContent-Length: 0\r\n\r\n") + conn.close() + + t = threading.Thread(target=serve) + t.start() + c = racer.Client(socket_path=path) + with pytest.raises(racer.StatusError): + call(c) + t.join(5) + lst.close() + os.unlink(path) + os.rmdir(d) + return got + + +def test_host_read_request_bytes(): + req = racer.Request(bytes.fromhex(KEY)) + with racer.HostBuffer(4196) as buf: + got = _capture( + "host", + lambda c: c.host_read(req, buf.fileno(), 4096, 100, offset=1, length=3, if_match='"v1"'), + ) + assert got["head"] == ( + f"POST /v1/objects/{KEY}/host-read HTTP/1.1\r\nHost: racer\r\nContent-Length: 0\r\n" + 'Range: bytes=1-3\r\nIf-Match: "v1"\r\n' + "Racer-Destination-Offset: 4096\r\nRacer-Destination-Length: 100\r\n\r\n" + ).encode() + # HostSink::open needs a writable, non-append descriptor. + (flags,) = got["flags"] + assert flags & os.O_ACCMODE == os.O_RDWR and not flags & os.O_APPEND + + +def test_gpu_read_request_bytes(): + req = racer.Request(bytes.fromhex(KEY)) + with racer.HostBuffer(1) as buf: + got = _capture("gpu", lambda c: c.gpu_read(req, buf.fileno(), 4196, 4096, 100)) + assert got["head"] == ( + f"POST /v1/objects/{KEY}/gpu-read HTTP/1.1\r\nHost: racer\r\nContent-Length: 0\r\n" + "Racer-Destination-Offset: 4096\r\nRacer-Destination-Length: 100\r\nRacer-Allocation-Size: 4196\r\n\r\n" + ).encode() + assert len(got["flags"]) == 1 + + +def test_direct_head_from_rust_is_accepted(): + # direct_head for a 3-byte range of an object of 2 pages + 5 bytes: + # the Get head with Content-Length 21 * (pages + 1) and Racer-Delivery + # inserted before Connection. + length = 2 * wire.PAGE_SIZE + 5 + raw = ( + 'HTTP/1.1 200 OK\r\nETag: "v1"\r\nRacer-Expires-At: 1700000000000\r\n' + "Content-Type: application/octet-stream\r\nRacer-Object-Length: %d\r\n" + "Racer-Range-Start: 3\r\nRacer-Range-End: 6\r\nContent-Length: 42\r\n" + "Racer-Delivery: host\r\nConnection: close\r\n\r\n" % length + ).encode() + h = wire.parse_response_head(raw) + from racer.client import Client + + meta, first, end = Client._check_selection(h, 3, 3, None) + assert (meta.size, first, end) == (length, 3, 6) + assert wire.decimal(h.get("content-length")) == wire.FRAME_SIZE * (wire.page_count(first, end) + 1) + seq = wire.Sequence(first, end) + seq.accept(wire.decode_frame(wire.encode_frame(1, 0, 3, 3))) + seq.accept(wire.decode_frame(wire.encode_frame(2, 1, 3, 0))) + assert seq.complete From 6239141bf4e8b1279727aaa79a18b8323ea77951 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 19:54:32 +0000 Subject: [PATCH 28/87] feat(racer): parse RACER_ENABLE_GPU and RACER_GPU_STAGING_BYTES --- cmd/racer-dataplane/src/config.rs | 82 +++++++++++++++++++++++++ cmd/racer-dataplane/src/test_support.rs | 2 + 2 files changed, 84 insertions(+) diff --git a/cmd/racer-dataplane/src/config.rs b/cmd/racer-dataplane/src/config.rs index f786f0be5..f409f92e2 100644 --- a/cmd/racer-dataplane/src/config.rs +++ b/cmd/racer-dataplane/src/config.rs @@ -121,6 +121,12 @@ pub struct Config { /// Opt into logical-CPU sizing instead of one eligible CPU per physical core. pub allow_smt: bool, pub enable_rdma: bool, + /// `gpu-read` support. Resolved against hardware at startup, on the GPU + /// helper thread, because GPU handles cannot move between threads. + pub enable_gpu: GpuMode, + /// Per-GPU cap on dataplane-owned ciphertext staging memory. Parsed and + /// validated now; plaintext `gpu-read` does not allocate staging. + pub gpu_staging_bytes: u64, /// Opt into experimental opaque HTTP transit; materialized relay is the default. pub opaque_relay: bool, /// Peer-only TCP_NODELAY. Peer exchanges are strictly request/response with @@ -187,6 +193,18 @@ impl Config { "false" => false, _ => return Err(Error::InvalidConfiguration), }; + let enable_gpu = match raw.enable_gpu.as_deref().unwrap_or("auto") { + "auto" => GpuMode::Auto, + "true" => GpuMode::Required, + "false" => GpuMode::Disabled, + _ => return Err(Error::InvalidConfiguration), + }; + let gpu_staging_bytes = raw.gpu_staging_bytes.unwrap_or(DEFAULT_GPU_STAGING_BYTES); + if gpu_staging_bytes < GPU_STAGING_UNIT + || !gpu_staging_bytes.is_multiple_of(GPU_STAGING_UNIT) + { + return Err(Error::InvalidConfiguration); + } let mut config = Self { warm: crate::read::warm::Config { max_ttl: Duration::from_secs(raw.warm_max_ttl_secs.unwrap_or(3600)), @@ -239,6 +257,8 @@ impl Config { crypto_helpers: raw.crypto_helpers.map(to_usize).transpose()?, allow_smt: raw.allow_smt.unwrap_or(false), enable_rdma, + enable_gpu, + gpu_staging_bytes, opaque_relay: raw.opaque_relay.unwrap_or(false), peer_tcp_nodelay: raw.peer_tcp_nodelay.unwrap_or(true), memory_pressure: raw.memory_pressure.unwrap_or(true), @@ -580,11 +600,31 @@ impl Config { } } +/// How `RACER_ENABLE_GPU` asks for `gpu-read`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GpuMode { + /// Serve `gpu-read` when built with `cuda` and a usable GPU is found. + Auto, + /// Fail startup unless a usable GPU is found. + Required, + /// Never serve `gpu-read`; answer 501. + Disabled, +} + +/// Default `RACER_GPU_STAGING_BYTES`: 256 MiB per GPU. +pub const DEFAULT_GPU_STAGING_BYTES: u64 = 256 * 1024 * 1024; + +/// GPU staging is allocated in 2 MiB device granules, so the cap must be a +/// whole number of them. +pub const GPU_STAGING_UNIT: u64 = 2 * 1024 * 1024; + /// Environment names are exact and case sensitive, even though envy folds keys. const ENV_NAMES: &[&str] = &[ "CLUSTER_ID", "CONTROL_ENDPOINT", "ENABLE_RDMA", + "ENABLE_GPU", + "GPU_STAGING_BYTES", "ALLOW_SMT", "OPAQUE_RELAY", "PEER_TCP_NODELAY", @@ -660,6 +700,9 @@ struct RawConfig { cluster_id: String, control_endpoint: String, enable_rdma: Option, + enable_gpu: Option, + #[serde(default, deserialize_with = "decimal")] + gpu_staging_bytes: Option, allow_smt: Option, opaque_relay: Option, peer_tcp_nodelay: Option, @@ -1323,6 +1366,45 @@ mod tests { assert!(parse(&[("RACER_ENABLE_RDMA", "true")]).unwrap().enable_rdma); } + #[test] + fn gpu_mode_and_staging_bytes_parse_and_validate() { + let config = parse(&[]).unwrap(); + assert_eq!(config.enable_gpu, GpuMode::Auto); + assert_eq!(config.gpu_staging_bytes, 256 << 20); + for (value, mode) in [ + ("auto", GpuMode::Auto), + ("true", GpuMode::Required), + ("false", GpuMode::Disabled), + ] { + assert_eq!( + parse(&[("RACER_ENABLE_GPU", value)]).unwrap().enable_gpu, + mode + ); + } + for value in ["TRUE", "1", "yes", "", " auto"] { + assert!(parse(&[("RACER_ENABLE_GPU", value)]).is_err(), "{value:?}"); + } + assert_eq!( + parse(&[("RACER_GPU_STAGING_BYTES", "4194304")]) + .unwrap() + .gpu_staging_bytes, + 4 << 20 + ); + for value in [ + "0", + "1048576", + "3145728", + "-2097152", + "2MiB", + "18446744073709551616", + ] { + assert!( + parse(&[("RACER_GPU_STAGING_BYTES", value)]).is_err(), + "{value:?}" + ); + } + } + #[test] fn native_auto_requires_build_and_available_library() { for requested in [None, Some("auto"), Some("false")] { diff --git a/cmd/racer-dataplane/src/test_support.rs b/cmd/racer-dataplane/src/test_support.rs index 74c3dbd72..1c2537d1e 100644 --- a/cmd/racer-dataplane/src/test_support.rs +++ b/cmd/racer-dataplane/src/test_support.rs @@ -310,6 +310,8 @@ pub mod cluster { memory_pressure: true, pprof_enabled: false, enable_rdma, + enable_gpu: crate::config::GpuMode::Disabled, + gpu_staging_bytes: crate::config::DEFAULT_GPU_STAGING_BYTES, control_endpoint: "https://control.invalid".into(), peer_listen: "127.0.0.1:0".parse().unwrap(), diagnostics_listen: "127.0.0.1:0".parse().unwrap(), From ace6881a1510abdeecff226c1269556a7b526eeb Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 19:57:57 +0000 Subject: [PATCH 29/87] feat(racer): serve gpu-read through a process-wide GPU helper thread --- cmd/racer-dataplane/src/app.rs | 15 +- cmd/racer-dataplane/src/client/direct.rs | 7 + cmd/racer-dataplane/src/client/listener.rs | 7 + cmd/racer-dataplane/src/gpu.rs | 542 +++++++++++++++++++++ cmd/racer-dataplane/src/gpu/tests.rs | 1 + cmd/racer-dataplane/src/lib.rs | 1 + 6 files changed, 569 insertions(+), 4 deletions(-) create mode 100644 cmd/racer-dataplane/src/gpu.rs create mode 100644 cmd/racer-dataplane/src/gpu/tests.rs diff --git a/cmd/racer-dataplane/src/app.rs b/cmd/racer-dataplane/src/app.rs index c8752a381..3041a76e7 100644 --- a/cmd/racer-dataplane/src/app.rs +++ b/cmd/racer-dataplane/src/app.rs @@ -131,6 +131,8 @@ pub struct Application { /// graph crosses a thread. Worker zero alone drives enrollment/control reloads. pub struct NodeState { devices: Option, + /// The process-wide GPU helper, when `gpu-read` is enabled. + gpu: Option>, inventory: Arc, /// Startup-fixed selector and claimed disks for `/debug/hardware`. @@ -209,6 +211,7 @@ impl NodeState { Ok(Self { peer_admission, devices: None, + gpu: None, peer_receive: Default::default(), inventory: Arc::new(crate::rdma::Inventory::default()), hardware: Default::default(), @@ -283,6 +286,8 @@ impl Application { self.config.limits.retained_snapshots.get(), ))); self.node.inventory.update(self.discovered_nics.clone())?; + Arc::get_mut(&mut self.node).ok_or(Error::Internal)?.gpu = + crate::gpu::GpuHelper::start(self.config.enable_gpu)?; if self.config.enable_rdma { self.node.native.place(&plan)?; self.node @@ -1196,10 +1201,12 @@ impl WorkerApplication { runtime.reactor.clone(), admission.clone(), )); - let responses = Rc::new( - Responses::new(io.clone(), delivery) - .with_observer(admission.policy().observer().clone()), - ); + let mut responses = Responses::new(io.clone(), delivery) + .with_observer(admission.policy().observer().clone()); + if let Some(gpu) = &node.gpu { + responses = responses.with_gpu(Rc::new(crate::gpu::GpuClient::new(gpu.clone()))); + } + let responses = Rc::new(responses); let clients = ClientListeners::new( coordinator, RequestParser::new(config.limits.header_bytes.get()), diff --git a/cmd/racer-dataplane/src/client/direct.rs b/cmd/racer-dataplane/src/client/direct.rs index 668443911..ef39bcb50 100644 --- a/cmd/racer-dataplane/src/client/direct.rs +++ b/cmd/racer-dataplane/src/client/direct.rs @@ -67,6 +67,13 @@ impl DirectPage<'_> { /// disconnects; its `Drop` must then keep any resource that a device may still /// write rather than free it. pub trait DirectSink { + /// Finish any setup that can still fail the request with an error status, + /// such as importing a GPU allocation. Awaited once before the read starts + /// and before the response head is sent. + fn ready<'a>(&'a mut self, _scope: &'a RequestScope) -> Operation<'a, ()> { + Box::pin(async { Ok(()) }) + } + /// Write `page` at `destination_offset` in the client's destination. The /// caller guarantees the bytes fall inside the request's destination /// window. Resolve only when the bytes are visible to the client. diff --git a/cmd/racer-dataplane/src/client/listener.rs b/cmd/racer-dataplane/src/client/listener.rs index d4af594ea..24ebef69a 100644 --- a/cmd/racer-dataplane/src/client/listener.rs +++ b/cmd/racer-dataplane/src/client/listener.rs @@ -969,6 +969,13 @@ async fn serve_connection( let sink = responses.open_sink(&request.kind, received.descriptors)?; Ok((request, sink)) }); + let parsed = match parsed { + Ok((request, Some(mut sink))) => match sink.ready(scope).await { + Ok(()) => Ok((request, Some(sink))), + Err(error) => Err(error), + }, + other => other, + }; let (request, sink) = match parsed { Ok(parsed) => parsed, Err(error) => { diff --git a/cmd/racer-dataplane/src/gpu.rs b/cmd/racer-dataplane/src/gpu.rs new file mode 100644 index 000000000..8b8304a3d --- /dev/null +++ b/cmd/racer-dataplane/src/gpu.rs @@ -0,0 +1,542 @@ +//! `gpu-read` backend: one GPU helper thread owns every CUDA object. +//! +//! `racer_gpu` types are `!Send`, so all GPU state lives on a single native +//! helper thread started once per process. I/O threads talk to it through a +//! job channel and get answers on one-shot channels, so they never block on +//! CUDA. The helper may block: it synchronizes the request's stream after each +//! page before it answers, which is what makes a page visible to the client. +//! +//! Each request gets its own stream and import. Jobs for one request run in +//! order, and a request has at most one job in flight, because the client path +//! awaits each page before the next. [`MAX_OPEN`] bounds requests. +//! +//! When a request ends without [`DirectSink::finish`] (for example the client +//! disconnected), dropping its sink queues a release job behind any page copy +//! still running. The helper releases the stream first, which synchronizes it, +//! then the import. `racer_gpu` leaks and counts anything whose release fails, +//! so memory a GPU may still write is never freed. +//! +//! Ciphertext pages (later phase) slot in as another [`Work`] variant: the +//! helper copies the sealed page into a per-GPU staging buffer and queues +//! [`racer_gpu::Stream::decrypt`] into the same import, then synchronizes and +//! checks the verdict before answering. + +use crate::client::direct::Destination; +use crate::client::direct::DirectPage; +use crate::client::direct::DirectSink; +use crate::client::direct::GpuService; +use crate::config::GpuMode; +use crate::error::Error; +use crate::error::Operation; +use crate::error::Result; +use crate::memory::VerifiedPage; +use crate::runtime::RequestScope; +use futures::channel::oneshot; +use racer_gpu::DeviceInfo; +use std::collections::HashMap; +use std::ops::Range; +use std::os::fd::OwnedFd; +use std::sync::Arc; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::sync::mpsc; +use std::task::Poll; + +/// Concurrent `gpu-read` requests per process. More get 503. +pub const MAX_OPEN: usize = 256; + +/// Builds the GPU backend on the helper thread. +pub type Loader = Box racer_gpu::Result + Send>; + +/// Plaintext bytes handed to the helper. The page is immutable and its +/// storage is reference counted, so the helper reads it in place; the handle +/// goes back with the answer so the page is released on the I/O thread. +struct Plaintext { + page: VerifiedPage, + range: Range, +} + +impl Plaintext { + fn bytes(&self) -> &[u8] { + &self.page.bytes()[self.range.clone()] + } +} + +/// Work for one request on the GPU. +enum Work { + /// Copy verified plaintext from host memory into the import. + Plaintext(Plaintext), +} + +/// A message to the helper thread. +enum Job { + Import { + id: u64, + fd: OwnedFd, + allocation_size: u64, + window: Range, + reply: oneshot::Sender>, + }, + Write { + id: u64, + offset: u64, + work: Work, + reply: oneshot::Sender<(Work, Result<()>)>, + }, + Release { + id: u64, + reply: Option>>, + }, + #[cfg(test)] + Run(Box), +} + +/// Handle to the GPU helper thread. Shared by every worker. +pub struct GpuHelper { + jobs: mpsc::Sender, + devices: Vec, + next: AtomicU64, + open: AtomicUsize, + leaked: Arc, +} + +impl GpuHelper { + /// Resolve `mode` against hardware. Returns `None` when GPU mode is off. + pub fn start(mode: GpuMode) -> Result>> { + match mode { + GpuMode::Disabled => Ok(None), + GpuMode::Auto if !cfg!(feature = "cuda") => { + eprintln!("racer-dataplane: stage=gpu state=disabled reason=not-built"); + Ok(None) + } + _ => Self::start_with(mode, Box::new(racer_gpu::Gpu::load)), + } + } + + /// Start the helper with a custom backend loader. + pub fn start_with(mode: GpuMode, loader: Loader) -> Result>> { + if mode == GpuMode::Disabled { + return Ok(None); + } + let (jobs, receiver) = mpsc::channel(); + let (ready, started) = mpsc::sync_channel(1); + let leaked = Arc::new(AtomicU64::new(0)); + let helper_leaked = leaked.clone(); + std::thread::Builder::new() + .name("racer-gpu".into()) + .spawn(move || helper_main(loader, receiver, ready, helper_leaked)) + .map_err(|_| Error::Unavailable)?; + let found = started.recv().map_err(|_| Error::Unavailable)?; + let devices = match found { + Ok(devices) if !devices.is_empty() => devices, + outcome => { + let reason = match outcome { + Err(error) => error.to_string(), + Ok(_) => "no-usable-gpu".into(), + }; + eprintln!("racer-dataplane: stage=gpu state=disabled reason={reason}"); + return if mode == GpuMode::Required { + Err(Error::Unavailable) + } else { + Ok(None) + }; + } + }; + eprintln!( + "racer-dataplane: stage=gpu state=enabled usable={}", + devices.len() + ); + Ok(Some(Arc::new(Self { + jobs, + devices, + next: AtomicU64::new(1), + open: AtomicUsize::new(0), + leaked, + }))) + } + + /// Usable GPUs found at startup. + pub fn devices(&self) -> &[DeviceInfo] { + &self.devices + } + + /// GPU resources leaked on purpose after a failed release. + pub fn leaked(&self) -> u64 { + self.leaked.load(Ordering::Relaxed) + } + + /// Requests that currently hold a sink. + pub fn open_requests(&self) -> usize { + self.open.load(Ordering::Relaxed) + } + + fn send(&self, job: Job) -> Result<()> { + self.jobs.send(job).map_err(|_| Error::Unavailable) + } + + /// Run `f` on the helper thread and wait for it. Tests use this to reach + /// thread-local simulation state. + #[cfg(test)] + pub(crate) fn run(&self, f: impl FnOnce() -> R + Send + 'static) -> R { + let (tx, rx) = mpsc::channel(); + self.send(Job::Run(Box::new(move || { + let _ = tx.send(f()); + }))) + .unwrap(); + rx.recv().unwrap() + } +} + +/// The per-worker [`GpuService`]. +pub struct GpuClient { + helper: Arc, +} + +impl GpuClient { + pub fn new(helper: Arc) -> Self { + Self { helper } + } +} + +impl GpuService for GpuClient { + fn open( + &self, + descriptor: OwnedFd, + destination: Destination, + allocation_size: u64, + ) -> Result> { + let window = destination.offset..destination.end(); + if window.start >= window.end || window.end > allocation_size { + return Err(Error::InvalidRequest); + } + let open = &self.helper.open; + if open.fetch_add(1, Ordering::AcqRel) >= MAX_OPEN { + open.fetch_sub(1, Ordering::AcqRel); + return Err(Error::Overloaded); + } + let mut sink = GpuSink { + helper: self.helper.clone(), + id: self.helper.next.fetch_add(1, Ordering::Relaxed), + window: window.clone(), + import: None, + released: false, + }; + let (reply, answer) = oneshot::channel(); + sink.helper.send(Job::Import { + id: sink.id, + fd: descriptor, + allocation_size, + window, + reply, + })?; + sink.import = Some(answer); + Ok(Box::new(sink)) + } +} + +/// One `gpu-read` request's view of its import on the helper. +struct GpuSink { + helper: Arc, + id: u64, + window: Range, + /// The pending import answer, until it has been checked. + import: Option>>, + released: bool, +} + +/// Wait for a helper answer, or stop early when the request is canceled. +/// Abandoning the wait never abandons the job: the helper still runs it. +async fn answer(receiver: &mut oneshot::Receiver, scope: &RequestScope) -> Result { + let registration = scope.cancellation.subscribe()?; + std::future::poll_fn(|cx| { + registration.register(cx.waker()); + if let Poll::Ready(result) = std::pin::Pin::new(&mut *receiver).poll(cx) { + return Poll::Ready(result.map_err(|_| Error::Unavailable)); + } + scope.check()?; + Poll::Pending + }) + .await +} + +use std::future::Future; + +impl GpuSink { + async fn imported(&mut self, scope: &RequestScope) -> Result<()> { + if let Some(receiver) = self.import.as_mut() { + let result = answer(receiver, scope).await?; + self.import = None; + result?; + } + Ok(()) + } +} + +impl DirectSink for GpuSink { + fn ready<'a>(&'a mut self, scope: &'a RequestScope) -> Operation<'a, ()> { + Box::pin(self.imported(scope)) + } + + fn deliver<'a>( + &'a mut self, + page: DirectPage<'a>, + destination_offset: u64, + scope: &'a RequestScope, + ) -> Operation<'a, ()> { + Box::pin(async move { + let end = destination_offset + .checked_add(page.len()) + .ok_or(Error::InvalidRange)?; + if destination_offset < self.window.start || end > self.window.end { + return Err(Error::InvalidRange); + } + self.imported(scope).await?; + let work = match page { + DirectPage::Plaintext(reader) => { + let slice = reader.slice(); + let start = slice.offset as usize; + Work::Plaintext(Plaintext { + page: reader.page().clone(), + range: start..start + slice.length as usize, + }) + } + }; + let (reply, mut receiver) = oneshot::channel(); + self.helper.send(Job::Write { + id: self.id, + offset: destination_offset, + work, + reply, + })?; + let (work, result) = answer(&mut receiver, scope).await?; + drop(work); + result + }) + } + + fn finish<'a>(&'a mut self, scope: &'a RequestScope) -> Operation<'a, ()> { + Box::pin(async move { + if self.released { + return Ok(()); + } + let (reply, mut receiver) = oneshot::channel(); + self.helper.send(Job::Release { + id: self.id, + reply: Some(reply), + })?; + self.released = true; + answer(&mut receiver, scope).await? + }) + } +} + +impl Drop for GpuSink { + fn drop(&mut self) { + if !self.released { + // Queued behind any copy still running; the helper syncs first. + let _ = self.helper.send(Job::Release { + id: self.id, + reply: None, + }); + } + self.helper.open.fetch_sub(1, Ordering::AcqRel); + } +} + +/// One request's GPU objects. Field order matters: the stream drops first, +/// which synchronizes it, then the import. +struct Request { + stream: racer_gpu::Stream, + import: racer_gpu::ImportedAllocation, +} + +/// State owned by the helper thread. +struct Helper { + gpu: racer_gpu::Gpu, + contexts: Vec, + requests: HashMap, + leaked: Arc, +} + +fn helper_main( + loader: Loader, + jobs: mpsc::Receiver, + ready: mpsc::SyncSender>>, + leaked: Arc, +) { + let mut helper = match discover(loader) { + Ok((gpu, contexts, usable)) => { + let _ = ready.send(Ok(usable)); + Helper { + gpu, + contexts, + requests: HashMap::new(), + leaked, + } + } + Err(error) => { + let _ = ready.send(Err(error)); + return; + } + }; + drop(ready); + while let Ok(job) = jobs.recv() { + helper.run(job); + } + // Every sender is gone: release what is left, stream first. + let ids: Vec = helper.requests.keys().copied().collect(); + for id in ids { + let _ = helper.release(id); + } +} + +type Discovered = (racer_gpu::Gpu, Vec, Vec); + +/// Load the backend, log every GPU, and retain a context on each usable one. +fn discover(loader: Loader) -> racer_gpu::Result { + let gpu = loader()?; + let mut contexts = Vec::new(); + let mut usable = Vec::new(); + for device in gpu.devices()? { + eprintln!( + "racer-dataplane: stage=gpu ordinal={} uuid={} pci_bus_id={} numa_node={} compute_mode={:?} usable={}", + device.ordinal, + device.uuid_string(), + device.pci_bus_id, + device.numa_node.map_or("unknown".into(), |n| n.to_string()), + device.compute_mode, + device.usable() + ); + if !device.usable() { + continue; + } + match gpu.context(device.ordinal) { + Ok(context) => { + contexts.push(context); + usable.push(device); + } + Err(error) => eprintln!( + "racer-dataplane: stage=gpu ordinal={} state=skipped reason={error}", + device.ordinal + ), + } + } + Ok((gpu, contexts, usable)) +} + +/// Map a GPU error on import to an HTTP-visible error. +fn import_error(error: racer_gpu::Error) -> Error { + match error { + racer_gpu::Error::WrongDevice => Error::NotImplemented, + racer_gpu::Error::InvalidRequest + | racer_gpu::Error::InvalidRange + | racer_gpu::Error::Driver(_) => Error::InvalidRequest, + racer_gpu::Error::OutOfMemory | racer_gpu::Error::Busy => Error::Overloaded, + _ => Error::Unavailable, + } +} + +impl Helper { + fn run(&mut self, job: Job) { + match job { + Job::Import { + id, + fd, + allocation_size, + window, + reply, + } => { + let result = self.import(id, fd, allocation_size, window); + self.note_leaks(); + let _ = reply.send(result); + } + Job::Write { + id, + offset, + work, + reply, + } => { + let result = self.write(id, offset, &work); + self.note_leaks(); + // If the I/O side stopped waiting, the page drops here. Its + // storage is shared and thread safe, so that is fine. + let _ = reply.send((work, result)); + } + Job::Release { id, reply } => { + let result = self.release(id); + if let Some(reply) = reply { + let _ = reply.send(result); + } + } + #[cfg(test)] + Job::Run(f) => f(), + } + } + + /// Try each usable GPU until one owns the allocation. + fn import(&mut self, id: u64, fd: OwnedFd, size: u64, window: Range) -> Result<()> { + let mut last = racer_gpu::Error::WrongDevice; + for context in &self.contexts { + let attempt = fd.try_clone().map_err(|_| Error::Unavailable)?; + match context.import(attempt, size, window.clone()) { + Ok(import) => { + let stream = context.stream().map_err(import_error)?; + self.requests.insert(id, Request { stream, import }); + return Ok(()); + } + Err(racer_gpu::Error::WrongDevice) => continue, + Err(error) => { + last = error; + break; + } + } + } + Err(import_error(last)) + } + + fn write(&mut self, id: u64, offset: u64, work: &Work) -> Result<()> { + let request = self.requests.get(&id).ok_or(Error::Internal)?; + let destination = racer_gpu::Destination::Import(&request.import); + let queued = match work { + Work::Plaintext(plaintext) => { + request + .stream + .copy_from_host(destination, offset, plaintext.bytes()) + } + }; + // Synchronize even after a failed queue call: part of it may run. + let synchronized = request.stream.synchronize(); + queued.and(synchronized).map_err(|error| match error { + racer_gpu::Error::InvalidRange => Error::InvalidRange, + _ => Error::Unavailable, + }) + } + + /// Drop the stream (which synchronizes it), then the import. + fn release(&mut self, id: u64) -> Result<()> { + let Some(request) = self.requests.remove(&id) else { + return Ok(()); + }; + let before = self.gpu.leaked(); + let synchronized = request.stream.synchronize(); + let Request { stream, import } = request; + drop(stream); + drop(import); + self.note_leaks(); + if synchronized.is_err() || self.gpu.leaked() != before { + return Err(Error::Unavailable); + } + Ok(()) + } + + fn note_leaks(&self) { + let leaked = self.gpu.leaked(); + if self.leaked.swap(leaked, Ordering::Relaxed) != leaked { + eprintln!("racer-dataplane: stage=gpu state=leaked total={leaked}"); + } + } +} + +#[cfg(test)] +mod tests; diff --git a/cmd/racer-dataplane/src/gpu/tests.rs b/cmd/racer-dataplane/src/gpu/tests.rs new file mode 100644 index 000000000..4563e55b7 --- /dev/null +++ b/cmd/racer-dataplane/src/gpu/tests.rs @@ -0,0 +1 @@ +use super::*; diff --git a/cmd/racer-dataplane/src/lib.rs b/cmd/racer-dataplane/src/lib.rs index af6090572..4b4014fa9 100644 --- a/cmd/racer-dataplane/src/lib.rs +++ b/cmd/racer-dataplane/src/lib.rs @@ -12,6 +12,7 @@ pub mod client; pub mod config; pub mod control; pub mod error; +pub mod gpu; pub mod hardware; pub mod http; pub mod memory; From 44d23d36ed02f9da0e95aafae75420c3bc4cceb8 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 20:02:34 +0000 Subject: [PATCH 30/87] feat(racer): test gpu-read end to end on simulated GPUs --- .../src/client/tests/direct.rs | 2 + .../src/client/tests/direct/gpu.rs | 239 ++++++++++++++++++ cmd/racer-dataplane/src/gpu.rs | 3 - cmd/racer-dataplane/src/gpu/tests.rs | 1 - 4 files changed, 241 insertions(+), 4 deletions(-) create mode 100644 cmd/racer-dataplane/src/client/tests/direct/gpu.rs delete mode 100644 cmd/racer-dataplane/src/gpu/tests.rs diff --git a/cmd/racer-dataplane/src/client/tests/direct.rs b/cmd/racer-dataplane/src/client/tests/direct.rs index bafbbf25b..07008d63f 100644 --- a/cmd/racer-dataplane/src/client/tests/direct.rs +++ b/cmd/racer-dataplane/src/client/tests/direct.rs @@ -507,3 +507,5 @@ fn gpu_read_delivers_pages_through_the_backend_seam() { settle(&fixture); assert_eq!(memfd.open_copies(), 1); } + +mod gpu; diff --git a/cmd/racer-dataplane/src/client/tests/direct/gpu.rs b/cmd/racer-dataplane/src/client/tests/direct/gpu.rs new file mode 100644 index 000000000..39bd885db --- /dev/null +++ b/cmd/racer-dataplane/src/client/tests/direct/gpu.rs @@ -0,0 +1,239 @@ +//! `gpu-read` end to end over the client socket, on simulated GPUs. +use super::*; +use crate::config::GpuMode; +use crate::gpu::GpuClient; +use crate::gpu::GpuHelper; +use racer_gpu::simulation::Environment; +use racer_gpu::simulation::Live; +use racer_gpu::simulation::Operation; +use racer_gpu::simulation::SimDevice; +use racer_gpu::simulation::Simulation; +use racer_gpu::simulation::read_allocation; + +const GRANULE: u64 = racer_gpu::simulation::GRANULARITY; + +thread_local! { + /// The simulation, on the GPU helper thread only. + static SIM: RefCell> = const { RefCell::new(None) }; +} + +fn sim() -> Simulation { + SIM.with(|sim| sim.borrow().as_ref().unwrap().0.clone()) +} + +/// A helper backed by `count` simulated GPUs. +fn start(mode: GpuMode, count: u8) -> Option> { + let devices: Vec = (0..count).map(SimDevice::new).collect(); + GpuHelper::start_with( + mode, + Box::new(move || { + let simulation = Simulation::new(devices); + let environment = simulation.enter(); + SIM.with(|sim| *sim.borrow_mut() = Some((simulation, environment))); + racer_gpu::Gpu::load() + }), + ) + .unwrap() +} + +fn allocate(helper: &GpuHelper, ordinal: i32, size: u64) -> OwnedFd { + helper.run(move || sim().allocate(ordinal, size)) +} + +fn live(helper: &GpuHelper) -> Live { + helper.run(|| sim().live()) +} + +/// No request holds GPU resources. Contexts stay retained by the helper. +fn assert_released(helper: &GpuHelper) { + let live = live(helper); + assert_eq!((live.imports, live.streams), (0, 0), "{live:?}"); + assert_eq!(helper.open_requests(), 0); + assert_eq!(helper.leaked(), 0); +} + +fn gpu_fixture(body: Vec, helper: &Arc) -> Fixture { + direct_fixture(body, Some(Rc::new(GpuClient::new(helper.clone())))) +} + +fn fields(range: &str, offset: u64, length: u64, size: u64) -> String { + format!( + "{range}Racer-Destination-Offset: {offset}\r\nRacer-Destination-Length: {length}\r\n\ + Racer-Allocation-Size: {size}\r\n" + ) +} + +#[test] +fn gpu_read_writes_the_range_at_the_destination_offset() { + let helper = start(GpuMode::Required, 2).unwrap(); + assert_eq!(helper.devices().len(), 2); + let fixture = gpu_fixture(b"hello".to_vec(), &helper); + // The allocation is on the second GPU, so the helper must find it. + let allocation = allocate(&helper, 1, GRANULE); + let output = exchange( + &fixture, + &direct_request( + "gpu-read", + &fields("Range: bytes=1-3\r\nIf-Match: \"v1\"\r\n", 100, 3, GRANULE), + ), + &[allocation.as_raw_fd()], + ); + let (head, body) = split(&output); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert!(head.contains("racer-delivery: gpu\r\n"), "{head}"); + assert!(head.contains("content-length: 42\r\n"), "{head}"); + assert_eq!(frames(body), [(1, 0, 1, 3), (2, 1, 3, 0)]); + assert_eq!(read_allocation(&allocation, 99, 5), b"\0ell\0"); + settle(&fixture); + assert_released(&helper); + assert_no_body_leases(&fixture); +} + +#[test] +fn gpu_read_spans_pages() { + let length = 2 * PAGE_BYTES as usize + 7; + let body: Vec = (0..length).map(|i| (i % 251) as u8).collect(); + let helper = start(GpuMode::Auto, 1).unwrap(); + let fixture = gpu_fixture(body.clone(), &helper); + let size = (length as u64 + 10).next_multiple_of(GRANULE); + let allocation = allocate(&helper, 0, size); + let output = exchange( + &fixture, + &direct_request("gpu-read", &fields("", 10, length as u64, size)), + &[allocation.as_raw_fd()], + ); + let (head, body_frames) = split(&output); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert_eq!( + frames(body_frames), + [ + (1, 0, 0, PAGE_BYTES as u32), + (1, 1, PAGE_BYTES, PAGE_BYTES as u32), + (1, 2, 2 * PAGE_BYTES, 7), + (2, 3, length as u64, 0) + ] + ); + assert!(read_allocation(&allocation, 10, length) == body); + settle(&fixture); + assert_released(&helper); +} + +#[test] +fn gpu_read_errors_before_the_head_release_everything() { + let helper = start(GpuMode::Required, 1).unwrap(); + let fixture = gpu_fixture(b"hello".to_vec(), &helper); + let allocation = allocate(&helper, 0, GRANULE); + // On a GPU the dataplane did not discover. + let foreign = allocate(&helper, 3, GRANULE); + for (fields, fd, status) in [ + ( + fields("If-Match: \"v2\"\r\n", 0, 5, GRANULE), + &allocation, + "412", + ), + // Larger than the real allocation. + (fields("", 0, 5, 2 * GRANULE), &allocation, "400"), + // Not a whole number of granules. + (fields("", 0, 5, GRANULE - 1), &allocation, "400"), + (fields("", 0, 5, GRANULE), &foreign, "501"), + ] { + let output = exchange( + &fixture, + &direct_request("gpu-read", &fields), + &[fd.as_raw_fd()], + ); + let (head, body) = split(&output); + assert!( + head.starts_with(&format!("http/1.1 {status}")), + "{fields} {head}" + ); + assert!(body.is_empty()); + settle(&fixture); + assert_released(&helper); + } + assert_eq!( + read_allocation(&allocation, 0, 5), + [0; 5], + "nothing written" + ); + assert_no_body_leases(&fixture); +} + +#[test] +fn gpu_read_driver_fault_fails_the_read_and_releases_the_import() { + let length = 2 * PAGE_BYTES as usize; + let helper = start(GpuMode::Required, 1).unwrap(); + let fixture = gpu_fixture(vec![b'x'; length], &helper); + let size = (length as u64).next_multiple_of(GRANULE); + let allocation = allocate(&helper, 0, size); + helper.run(|| sim().fault(Operation::CopyH2d, -5)); + let output = exchange( + &fixture, + &direct_request("gpu-read", &fields("", 0, length as u64, size)), + &[allocation.as_raw_fd()], + ); + let (head, body) = split(&output); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert!( + frames(body).iter().all(|frame| frame.0 != 2), + "no kind 2 frame after a failure" + ); + settle(&fixture); + assert_released(&helper); + fixture.listeners.admission.reclaim_buffers(); + assert_no_body_leases(&fixture); +} + +#[test] +fn gpu_read_client_disconnect_releases_the_import() { + let length = 3 * PAGE_BYTES as usize + 1; + let helper = start(GpuMode::Required, 1).unwrap(); + let fixture = gpu_fixture(vec![b'x'; length], &helper); + let size = (length as u64).next_multiple_of(GRANULE); + let allocation = allocate(&helper, 0, size); + let mut socket = fixture.connect(); + send_with_fds( + &socket, + &direct_request("gpu-read", &fields("", 0, length as u64, size)), + &[allocation.as_raw_fd()], + ); + let output = fixture.receive(&mut socket, false); + assert!(split(&output).0.starts_with("http/1.1 200")); + drop(socket); + settle(&fixture); + assert_released(&helper); + fixture.listeners.admission.reclaim_buffers(); + assert_no_body_leases(&fixture); +} + +#[test] +fn gpu_mode_resolves_against_hardware() { + assert!(start(GpuMode::Disabled, 1).is_none()); + assert!(start(GpuMode::Auto, 0).is_none()); + assert!( + GpuHelper::start_with( + GpuMode::Required, + Box::new(|| Err(racer_gpu::Error::Unavailable)) + ) + .is_err() + ); + let unusable = SimDevice { + compute_mode: 3, + ..SimDevice::new(0) + }; + assert!( + GpuHelper::start_with( + GpuMode::Required, + Box::new(move || { + let simulation = Simulation::new(vec![unusable]); + std::mem::forget(simulation.enter()); + racer_gpu::Gpu::load() + }) + ) + .is_err() + ); + // Without the cuda feature, auto never loads a backend. + if !cfg!(feature = "cuda") { + assert!(GpuHelper::start(GpuMode::Auto).unwrap().is_none()); + } +} diff --git a/cmd/racer-dataplane/src/gpu.rs b/cmd/racer-dataplane/src/gpu.rs index 8b8304a3d..0fb529ba2 100644 --- a/cmd/racer-dataplane/src/gpu.rs +++ b/cmd/racer-dataplane/src/gpu.rs @@ -537,6 +537,3 @@ impl Helper { } } } - -#[cfg(test)] -mod tests; diff --git a/cmd/racer-dataplane/src/gpu/tests.rs b/cmd/racer-dataplane/src/gpu/tests.rs deleted file mode 100644 index 4563e55b7..000000000 --- a/cmd/racer-dataplane/src/gpu/tests.rs +++ /dev/null @@ -1 +0,0 @@ -use super::*; From a78ec79049815af658015fd374a2449c469a03e9 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 20:02:46 +0000 Subject: [PATCH 31/87] feat(racer): document gpu-read and GPU configuration --- deploy/racer/dataplane-config.yaml.tmpl | 3 ++- docs/content/reference/racer.md | 26 ++++++++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/deploy/racer/dataplane-config.yaml.tmpl b/deploy/racer/dataplane-config.yaml.tmpl index 01c385369..6a90f0cfe 100644 --- a/deploy/racer/dataplane-config.yaml.tmpl +++ b/deploy/racer/dataplane-config.yaml.tmpl @@ -16,7 +16,8 @@ data: # GPU reads need an image built with RACER_NATIVE_CUDA=true. auto turns GPU # mode off when no usable GPU is found; true fails startup instead. # RACER_ENABLE_GPU: "auto" - # Optional cap on ciphertext staging memory per GPU, in bytes. + # Cap on ciphertext staging memory per GPU, in bytes: a nonzero multiple of + # 2 MiB, default 256 MiB. Reserved for GPU decryption; not used yet. # RACER_GPU_STAGING_BYTES: "268435456" # No thread cap by default: eligible physical cores, CPU quota, and resource budgets apply. # I/O shards share NUMA-local crypto threads at approximately 2:1, using leftovers: diff --git a/docs/content/reference/racer.md b/docs/content/reference/racer.md index 2cc66f9eb..9b74eb661 100644 --- a/docs/content/reference/racer.md +++ b/docs/content/reference/racer.md @@ -203,7 +203,7 @@ Numeric values are unsigned decimal strings: `268435456`, not `256Mi`. | `RACER_ALLOW_SMT` | `false` | Use eligible physical cores by default; `true` opts into logical CPUs | | `RACER_ENABLE_RDMA` | `auto` | `auto`, `true`, or `false`; see [modes](#rdma) | | `RACER_ENABLE_GPU` | `auto` | `auto`, `true`, or `false`; see [GPU reads](#gpu-reads) | -| `RACER_GPU_STAGING_BYTES` | Unset (dataplane default) | Cap on ciphertext staging memory per GPU, in bytes | +| `RACER_GPU_STAGING_BYTES` | `268435456` (256 MiB) | Cap on ciphertext staging memory per GPU, in bytes. Must be a nonzero multiple of 2 MiB. Not used yet | | `RACER_SHARES` | `4` | Enrollment proposal; explicit Node `shares` wins | | `RACER_PLAINTEXT_BYTES` / `RACER_CIPHERTEXT_BYTES` | Unset: `67108864` (64 MiB) / `134217728` (128 MiB) per final I/O worker | Independent plaintext/ciphertext admission budgets; an explicit value is node-wide and divided among I/O workers | | `RACER_DIRTY_BYTES` | Unset: `167772160` (160 MiB) per final I/O worker | Dirty-data budget; an explicit value is node-wide | @@ -384,8 +384,19 @@ Racer-Destination-Length: - If the read fails partway, the connection closes without the completion frame and the file may hold partial data. -`POST /v1/objects//gpu-read` is reserved for GPU memory. This dataplane -answers it with 501. +`POST /v1/objects//gpu-read` writes into GPU memory. It works like +`host-read`, with these differences: + +- The descriptor comes from `cuMemExportToShareableHandle` with the POSIX file + descriptor handle type. `Racer-Allocation-Size` is required and must be the + size passed to `cuMemCreate`. +- The answer is 501 when GPU mode is off or the memory is on a GPU the + dataplane did not find. It is 400 when the size does not match the memory or + the window does not fit in it. +- A success has `Racer-Delivery: gpu`. A page frame means the GPU copy of that + page is complete. +- When the read ends for any reason, the dataplane waits for its GPU work, + then releases the memory. See [GPU reads](#gpu-reads). ### Warm and hard retention @@ -1006,8 +1017,13 @@ GPU memory. They need: - An NVIDIA node set up by the unbounded agent, which installs the NVIDIA container runtime and a CDI spec. - `RACER_ENABLE_GPU` in `racer-dataplane-config`. `auto` turns GPU mode off - when no usable GPU is found. `true` makes startup fail instead. `false` - turns it off. + when no usable GPU is found, or when the image was built without CUDA. `true` + makes startup fail instead. `false` turns it off. + +At startup the dataplane logs each GPU it finds: ordinal, UUID, PCI address, +NUMA node, compute mode, and whether it can use it. It skips GPUs in +`EXCLUSIVE_PROCESS` or `PROHIBITED` mode. One helper thread does all GPU work +for the process. At most 256 GPU reads run at once; more get 503. The dataplane does not request `nvidia.com/gpu`. It is privileged, so it already sees `/dev/nvidia*`. It only needs the host driver libraries. From 0fce9d9b8928c9e2b84b031708dc51ab8f8de17f Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 20:20:39 +0000 Subject: [PATCH 32/87] feat(racer): simulate cuFile reads of O_DIRECT files --- cmd/racer-dataplane/gpu/src/simulation.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/racer-dataplane/gpu/src/simulation.rs b/cmd/racer-dataplane/gpu/src/simulation.rs index 16b671753..bb6676ff8 100644 --- a/cmd/racer-dataplane/gpu/src/simulation.rs +++ b/cmd/racer-dataplane/gpu/src/simulation.rs @@ -730,13 +730,16 @@ unsafe extern "C" fn file(fd: c_int, out: ffi::Out) -> c_int { if !w.file_driver { return (code::EUNAVAILABLE, 0); } - let duplicate = unsafe { libc::fcntl(fd, libc::F_DUPFD_CLOEXEC, 0) }; - if fd < 0 || duplicate < 0 { + if fd < 0 || unsafe { libc::fcntl(fd, libc::F_GETFD) } < 0 { return (code::EINVAL, 0); } + // cuFile reads with its own alignment rules. Reopen the same file + // without O_DIRECT so host reads of any length and buffer work. + let Ok(reopened) = File::open(format!("/proc/self/fd/{fd}")) else { + return (code::EINVAL, 0); + }; let handle = give(w, out); - w.files - .insert(handle, unsafe { File::from_raw_fd(duplicate) }); + w.files.insert(handle, reopened); (code::OK, handle) }) } From 4702141087c3c31b33b657f6326126569b92975b Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 20:20:40 +0000 Subject: [PATCH 33/87] feat(racer): locate v2 disk ciphertext for direct reads and derive GPU page keys --- cmd/racer-dataplane/src/security.rs | 16 ++++ cmd/racer-dataplane/src/store.rs | 135 ++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+) diff --git a/cmd/racer-dataplane/src/security.rs b/cmd/racer-dataplane/src/security.rs index ff3f09493..627ae7171 100644 --- a/cmd/racer-dataplane/src/security.rs +++ b/cmd/racer-dataplane/src/security.rs @@ -317,6 +317,22 @@ impl PageCrypto { self.buffers = Some(buffers); self } + /// Derive the AES-256-GCM key and nonce that XAES-256-GCM uses for this + /// page, so a GPU can authenticate and decrypt it. Checks the cache, key ID + /// and purpose exactly as [`Self::decrypt`] does. The derived key is valid + /// for this one page nonce and is wiped on drop. Three AES blocks, so it + /// runs on the I/O thread rather than queueing on the crypto worker. + pub(crate) fn derive_page_key( + &self, + envelope: &PageEnvelope, + ) -> Result { + let cache = &envelope.page.version.object.cache; + let key = self + .keys + .lease(Some(cache), envelope.key_id, KeyPurpose::Page)?; + key.derive_page_subkey(cache, envelope.key_id, &envelope.nonce.0) + .map_err(|_| Error::MissingKey) + } /// Authenticate the entire page before constructing a publishable lease. /// Futures stay on I/O, even though inputs/completions are Send: /// ```compile_fail diff --git a/cmd/racer-dataplane/src/store.rs b/cmd/racer-dataplane/src/store.rs index 972d882d7..7717fd148 100644 --- a/cmd/racer-dataplane/src/store.rs +++ b/cmd/racer-dataplane/src/store.rs @@ -293,6 +293,141 @@ impl StoreReader { .map(|(copy, _)| copy)) }) } + + /// Locate a published v2 record's ciphertext on its device for a direct + /// (cuFile) read, reading and validating only the header through host + /// memory. Returns None on a miss, for v1 records, and for any record + /// this read path would also treat as a miss. The ciphertext checksum is + /// not verified here: the consumer must check + /// [`DirectRecord::checksum`] before trusting the bytes. + pub(crate) fn direct_read<'a>( + &'a self, + page: &'a PageId, + scope: &'a RequestScope, + reserve: impl Fn(usize) -> Result> + 'a, + ) -> Operation<'a, Option> { + Box::pin(async move { + scope.check()?; + let Some(entry) = self.metrics.lookup( + crate::telemetry::LookupTier::DiskIndex, + self.index.lookup(page), + )? + else { + return Ok(None); + }; + let location = entry.location.clone(); + let token = ReadToken { + page: page.clone(), + location: location.clone(), + }; + if self + .segments + .validate(location.segment, location.generation, &location.extent) + .is_err() + { + self.invalidate(&token)?; + return Ok(None); + } + // One lease rides with the header read; the other keeps the + // generation alive until the caller's direct transfer completes. + let (Ok(read_lease), Ok(hold)) = ( + self.segments.lease(location.segment, location.generation), + self.segments.lease(location.segment, location.generation), + ) else { + self.invalidate(&token)?; + return Ok(None); + }; + let prefix = location.extent.length().min(MAX_HEADER_BYTES); + let Ok(prefix_extent) = Extent::new(location.extent.offset(), prefix) else { + return Ok(None); + }; + let mut reserved = reserve(prefix); + if matches!(reserved, Err(Error::Overloaded)) { + self.slabs.reclaim_idle(); + reserved = reserve(prefix); + } + let reservation = reserved?; + let Ok(buffer) = self.slabs.allocate(prefix, reservation) else { + return Ok(None); + }; + let buffer = match self + .slabs + .read(&self.reactor, prefix_extent, buffer, read_lease, scope) + .await + { + Ok(buffer) => buffer, + Err( + Error::Io | Error::Os(_) | Error::CorruptRecord | Error::InvalidConfiguration, + ) => { + return Ok(None); + } + Err(error) => return Err(error), + }; + let decoded = match parse_prefix(buffer.bytes()?, location.extent) { + Ok(decoded) => decoded, + // A short prefix of a valid v1 record, another version, or + // damage: the host read path classifies and reports it. + Err(_) => return Ok(None), + }; + drop(buffer); + if decoded.header.envelope.page != *page + || decoded.header.generation != location.generation + || decoded.header.metadata != entry.metadata + || decoded.header.envelope.key_id != entry.key_id + { + self.corrupt_miss(); + self.invalidate(&token)?; + return Ok(None); + } + if self.index.lookup(page)?.is_none_or(|current| { + current.location != location || current.key_id != entry.key_id + }) { + return Ok(None); + } + let Some(ciphertext) = direct_ciphertext(&self.slabs, &location, &hold, &decoded)? + else { + return Ok(None); + }; + // cuFile reads whole 4 KiB blocks. v2 extents are 4 KiB multiples + // and the ciphertext ends the record, so rounding up stays inside it. + let rounded = ciphertext.length.next_multiple_of(CIPHERTEXT_ALIGNMENT); + let read_length = if decoded.ciphertext.start + rounded <= location.extent.length() { + rounded + } else { + ciphertext.length + }; + self.clock.mark_read(location.segment)?; + self.index + .retention() + .disk_read(page, u64::from(entry.metadata.page_length(page)?)); + Ok(Some(DirectRecord { + token, + metadata: entry.metadata.for_pin(), + envelope: decoded.header.envelope, + checksum: decoded.checksum, + ciphertext, + read_length, + lease: hold, + })) + }) + } +} + +/// A published v2 record located for a direct read by [`StoreReader::direct_read`]. +pub(crate) struct DirectRecord { + /// Invalidates exactly this mapping if the bytes fail verification. + pub token: ReadToken, + pub metadata: crate::model::ObjectMetadata, + pub envelope: PageEnvelope, + /// The stored CRC-64 of the sealed page (ciphertext and tag). + pub checksum: u64, + pub ciphertext: DirectCiphertext, + /// Bytes to transfer from `ciphertext.offset`: the ciphertext length + /// rounded up to whole 4 KiB blocks when the record holds them. + pub read_length: usize, + /// Keeps the segment generation, and so the bytes, from being recycled. + /// Hold it until the direct transfer completes. + pub lease: SegmentLease, } struct Dirty { From 9cf7e03b9a97bf9bcad754c69f8ce95e6ddde3d5 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 20:20:40 +0000 Subject: [PATCH 34/87] feat(racer): deliver sealed pages to gpu-read and decrypt them on the GPU --- cmd/racer-dataplane/src/app.rs | 2 +- cmd/racer-dataplane/src/client.rs | 60 +- cmd/racer-dataplane/src/client/direct.rs | 26 +- .../src/client/tests/direct.rs | 4 +- .../src/client/tests/direct/gpu.rs | 6 +- cmd/racer-dataplane/src/gpu.rs | 585 ++++++++++++++++-- cmd/racer-dataplane/src/read.rs | 1 + cmd/racer-dataplane/src/read/dispatch.rs | 101 +++ cmd/racer-dataplane/src/read/fill.rs | 224 ++++++- cmd/racer-dataplane/src/read/range_stream.rs | 184 +++++- cmd/racer-dataplane/src/read/sealed.rs | 206 ++++++ 11 files changed, 1317 insertions(+), 82 deletions(-) create mode 100644 cmd/racer-dataplane/src/read/sealed.rs diff --git a/cmd/racer-dataplane/src/app.rs b/cmd/racer-dataplane/src/app.rs index 3041a76e7..2bf10220a 100644 --- a/cmd/racer-dataplane/src/app.rs +++ b/cmd/racer-dataplane/src/app.rs @@ -287,7 +287,7 @@ impl Application { ))); self.node.inventory.update(self.discovered_nics.clone())?; Arc::get_mut(&mut self.node).ok_or(Error::Internal)?.gpu = - crate::gpu::GpuHelper::start(self.config.enable_gpu)?; + crate::gpu::GpuHelper::start(self.config.enable_gpu, self.config.gpu_staging_bytes)?; if self.config.enable_rdma { self.node.native.place(&plan)?; self.node diff --git a/cmd/racer-dataplane/src/client.rs b/cmd/racer-dataplane/src/client.rs index 8c880e5b9..c9d052067 100644 --- a/cmd/racer-dataplane/src/client.rs +++ b/cmd/racer-dataplane/src/client.rs @@ -23,6 +23,7 @@ use crate::model::PageNumber; use crate::model::ResolvedRange; use crate::read::ReadPayload; use crate::read::ReadResponse; +use crate::read::range_stream::DirectItem; use crate::read::range_stream::RangeStream; use crate::runtime::RequestScope; use crate::security::Authorization; @@ -702,8 +703,13 @@ impl Responses { return Err(Error::InvalidRequest); } let mut first = if let ReadPayload::Stream(stream) = &mut response.payload { - match stream.next_slice().await { - Ok(Some(reader)) => Some(reader), + // A GPU sink that decrypts asks for sealed pages, so the CPU + // decrypts only pages it holds as plaintext anyway. + if let Some(target) = sink.sealed_target() { + stream.set_sealed(target)?; + } + match stream.next_direct().await { + Ok(Some(item)) => Some(item), result => { let error = result.err().unwrap_or(Error::BadGateway); self.observer.record( @@ -727,11 +733,11 @@ impl Responses { while sent < expected { let mut progress_scope = scope.clone(); progress_scope.deadline.0 = uring_runtime::environment::now() + timeout; - let reader = match first.take() { - Some(reader) => reader, + let item = match first.take() { + Some(item) => item, None => std::future::poll_fn(|cx| { progress_scope.check()?; - stream.next_slice().as_mut().poll(cx) + stream.next_direct().as_mut().poll(cx) }) .await .inspect_err(|&error| { @@ -745,7 +751,10 @@ impl Responses { })? .ok_or(Error::BadGateway)?, }; - let slice = reader.slice(); + let slice = match &item { + DirectItem::Plaintext(reader) => reader.slice(), + DirectItem::Sealed(_, slice) => *slice, + }; if range.slice_at(slice.page)? != Some(slice) || slice.length == 0 || u64::from(slice.length) > expected - sent @@ -763,10 +772,29 @@ impl Responses { .offset .checked_add(offset - range.start()) .ok_or(Error::BadGateway)?; + // A plaintext lease must outlive the write that borrows it. + let reader; { - let mut write = - sink.deliver(DirectPage::Plaintext(&reader), target, &progress_scope); - std::future::poll_fn(|cx| { + let (page, report) = match item { + DirectItem::Plaintext(lease) => { + reader = Some(lease); + (DirectPage::Plaintext(reader.as_ref().unwrap()), None) + } + DirectItem::Sealed(sealed, _) => { + reader = None; + let report = sealed.report(); + let start = u64::from(slice.offset); + ( + DirectPage::Sealed { + page: sealed, + range: start..start + u64::from(slice.length), + }, + Some(report), + ) + } + }; + let mut write = sink.deliver(page, target, &progress_scope); + let result = std::future::poll_fn(|cx| { if let Poll::Ready(result) = write.as_mut().poll(cx) { return Poll::Ready(result); } @@ -774,7 +802,19 @@ impl Responses { stream.poll_prefetch(cx); Poll::Pending }) - .await?; + .await; + drop(write); + if let Some(report) = report { + // The owner drops a copy the GPU rejected, as it would + // after a failed CPU decrypt, and may persist one it + // accepted. + match result { + Ok(()) => stream.report_sealed(report, true), + Err(Error::CorruptRecord) => stream.report_sealed(report, false), + Err(_) => {} + } + } + result?; } drop(reader); // The page is written: no client credit gates its release. diff --git a/cmd/racer-dataplane/src/client/direct.rs b/cmd/racer-dataplane/src/client/direct.rs index ef39bcb50..e6799ed5a 100644 --- a/cmd/racer-dataplane/src/client/direct.rs +++ b/cmd/racer-dataplane/src/client/direct.rs @@ -17,6 +17,8 @@ use crate::error::Operation; use crate::error::Result; use crate::http::Delivery; use crate::http::ReaderLease; +use crate::read::sealed::SealedPage; +use crate::read::sealed::SealedTarget; use crate::runtime::RequestScope; use http1::connection::ReceivedDescriptors; pub use racer_object_wire::client::Destination; @@ -32,15 +34,23 @@ pub const DIRECT_WINDOW_PAGES: usize = 4; /// One page handed to a [`DirectSink`]. /// -/// Today every page arrives as verified plaintext. A GPU backend that decrypts -/// on the device will add a variant that carries the page identity and asks the -/// sink to fetch ciphertext itself; the enum is non-exhaustive for that reason. +/// Pages arrive as verified plaintext unless the sink asked for sealed pages +/// with [`DirectSink::sealed_target`]. #[non_exhaustive] pub enum DirectPage<'a> { /// Verified plaintext already in memory. [`ReaderLease::slice_bytes`] holds /// exactly the bytes to write; [`ReaderLease::page`] can be cloned to keep /// the page alive past this call. Plaintext(&'a ReaderLease), + + /// A whole sealed page, for the sink to verify and decrypt. Only + /// `range`, a byte range of the page's plaintext, is written. A sink + /// must not write anything if verification fails, and must then return + /// [`Error::CorruptRecord`]. + Sealed { + page: Box, + range: std::ops::Range, + }, } impl DirectPage<'_> { @@ -48,6 +58,7 @@ impl DirectPage<'_> { pub fn len(&self) -> u64 { match self { Self::Plaintext(reader) => u64::from(reader.slice().length), + Self::Sealed { range, .. } => range.end.saturating_sub(range.start), } } @@ -86,6 +97,13 @@ pub trait DirectSink { /// Wait for all work that touches the destination, then release it. fn finish<'a>(&'a mut self, scope: &'a RequestScope) -> Operation<'a, ()>; + + /// Ask for sealed pages ([`DirectPage::Sealed`]) where the read path has + /// no plaintext in memory. Called after [`Self::ready`]. The default, + /// None, gets verified plaintext only. + fn sealed_target(&self) -> Option { + None + } } /// A GPU backend for `gpu-read`. Without one, `gpu-read` answers 501. @@ -159,6 +177,8 @@ impl DirectSink for HostSink { .write_slice_at(reader, self.file.clone(), destination_offset, scope) .await } + // Never requested: host sinks have no sealed target. + DirectPage::Sealed { .. } => Err(Error::Internal), } }) } diff --git a/cmd/racer-dataplane/src/client/tests/direct.rs b/cmd/racer-dataplane/src/client/tests/direct.rs index 07008d63f..155c77e04 100644 --- a/cmd/racer-dataplane/src/client/tests/direct.rs +++ b/cmd/racer-dataplane/src/client/tests/direct.rs @@ -442,7 +442,9 @@ impl DirectSink for FakeSink { destination_offset: u64, _scope: &'a RequestScope, ) -> crate::error::Operation<'a, ()> { - let DirectPage::Plaintext(reader) = page; + let DirectPage::Plaintext(reader) = page else { + panic!("fake sink asks for plaintext only"); + }; let bytes = reader.slice_bytes().to_vec(); let recorded = self.0.clone(); Box::pin(async move { diff --git a/cmd/racer-dataplane/src/client/tests/direct/gpu.rs b/cmd/racer-dataplane/src/client/tests/direct/gpu.rs index 39bd885db..dbe92b6de 100644 --- a/cmd/racer-dataplane/src/client/tests/direct/gpu.rs +++ b/cmd/racer-dataplane/src/client/tests/direct/gpu.rs @@ -234,6 +234,10 @@ fn gpu_mode_resolves_against_hardware() { ); // Without the cuda feature, auto never loads a backend. if !cfg!(feature = "cuda") { - assert!(GpuHelper::start(GpuMode::Auto).unwrap().is_none()); + assert!( + GpuHelper::start(GpuMode::Auto, crate::config::DEFAULT_GPU_STAGING_BYTES) + .unwrap() + .is_none() + ); } } diff --git a/cmd/racer-dataplane/src/gpu.rs b/cmd/racer-dataplane/src/gpu.rs index 0fb529ba2..4c1d85d60 100644 --- a/cmd/racer-dataplane/src/gpu.rs +++ b/cmd/racer-dataplane/src/gpu.rs @@ -16,10 +16,29 @@ //! then the import. `racer_gpu` leaks and counts anything whose release fails, //! so memory a GPU may still write is never freed. //! -//! Ciphertext pages (later phase) slot in as another [`Work`] variant: the -//! helper copies the sealed page into a per-GPU staging buffer and queues -//! [`racer_gpu::Stream::decrypt`] into the same import, then synchronizes and -//! checks the verdict before answering. +//! # Sealed pages +//! +//! Pages that are not verified plaintext in memory arrive sealed +//! ([`crate::read::sealed::SealedPage`]) and the GPU authenticates and decrypts +//! them. Each GPU has one staging buffer of [`SLOT_BYTES`] slots, allocated on +//! first use and capped by `RACER_GPU_STAGING_BYTES`: +//! +//! - Slot 0 holds host ciphertext copied in for the current job. +//! - Slot 1 holds plaintext when a range covers only part of a page: the GPU +//! decrypts the whole page there (the tag covers all of it), checks the +//! verdict, then copies only the requested bytes into the import. +//! - The other slots hold ciphertext that a page's owning worker read from a +//! local v2 disk record with cuFile ([`GpuHelper::stage_file`]). A +//! [`StagedCiphertext`] owns its slot and returns it on drop. When every +//! slot is taken, the owner reads the record through host memory instead. +//! +//! The helper runs one job at a time and synchronizes before it answers, so the +//! two scratch slots are never shared by two jobs and no staged slot is in use +//! by a stream between jobs, which [`racer_gpu::GpuFile::read`] requires. +//! +//! At startup the helper probes each GPU with a zero-length decrypt. A GPU +//! whose shim has no decrypt kernels, or whose staging cap cannot hold the two +//! scratch slots, gets plaintext pages only: the CPU decrypts as for a Get. use crate::client::direct::Destination; use crate::client::direct::DirectPage; @@ -30,6 +49,10 @@ use crate::error::Error; use crate::error::Operation; use crate::error::Result; use crate::memory::VerifiedPage; +use crate::model::PAGE_BYTES; +use crate::read::sealed::SealedPage; +use crate::read::sealed::SealedSource; +use crate::read::sealed::SealedTarget; use crate::runtime::RequestScope; use futures::channel::oneshot; use racer_gpu::DeviceInfo; @@ -37,6 +60,8 @@ use std::collections::HashMap; use std::ops::Range; use std::os::fd::OwnedFd; use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU64; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; @@ -46,6 +71,22 @@ use std::task::Poll; /// Concurrent `gpu-read` requests per process. More get 503. pub const MAX_OPEN: usize = 256; +/// Bytes per staging slot: one sealed page (plaintext and tag), rounded up +/// to whole 4 KiB blocks so cuFile reads land block aligned. +pub const SLOT_BYTES: u64 = (PAGE_BYTES + 16).next_multiple_of(4096); + +/// Scratch slots each job may use: host ciphertext and partial plaintext. +const SCRATCH_SLOTS: u64 = 2; + +/// Offset of the host ciphertext scratch slot. +const CIPHERTEXT_SCRATCH: u64 = 0; + +/// Offset of the partial plaintext scratch slot. +const PLAINTEXT_SCRATCH: u64 = SLOT_BYTES; + +/// Registered cuFile handles kept per process; more are reopened on demand. +const MAX_FILES: usize = 64; + /// Builds the GPU backend on the helper thread. pub type Loader = Box racer_gpu::Result + Send>; @@ -63,10 +104,97 @@ impl Plaintext { } } +/// A sealed page and the bytes of its plaintext the request wants. +struct Sealed { + page: Box, + range: Range, +} + /// Work for one request on the GPU. enum Work { /// Copy verified plaintext from host memory into the import. Plaintext(Plaintext), + + /// Verify and decrypt a sealed page into the import. + Sealed(Sealed), +} + +/// The staged-ciphertext slots of one GPU. Shared with every +/// [`StagedCiphertext`] so a slot comes back when its page is dropped, on any +/// thread, without a message to the helper. +struct SlotPool { + free: Mutex>, +} + +impl SlotPool { + fn take(&self) -> Option { + self.free.lock().ok()?.pop() + } + + fn put(&self, slot: u64) { + if let Ok(mut free) = self.free.lock() { + free.push(slot); + } + } + + fn available(&self) -> usize { + self.free.lock().map_or(0, |free| free.len()) + } +} + +/// Ciphertext in one staging slot of one GPU, read there by cuFile. Owns the +/// slot until dropped. Only the helper thread touches the bytes. +pub struct StagedCiphertext { + pool: Arc, + ordinal: i32, + slot: u64, +} + +impl StagedCiphertext { + /// The GPU the bytes are on. + pub fn ordinal(&self) -> i32 { + self.ordinal + } + + /// Byte offset of the slot in the GPU's staging buffer. + fn offset(&self) -> u64 { + self.slot * SLOT_BYTES + } +} + +impl Drop for StagedCiphertext { + fn drop(&mut self) { + self.pool.put(self.slot); + } +} + +/// Counters for GPU decryption, for logs and tests. +#[derive(Debug, Default)] +pub struct GpuCounters { + /// Sealed pages the GPU verified and wrote. + pub decrypted: AtomicU64, + + /// Sealed pages the GPU rejected (CRC or tag mismatch). + pub rejected: AtomicU64, + + /// Pages cuFile read straight into staging. + pub staged: AtomicU64, + + /// Direct reads that found no free staging slot. + pub exhausted: AtomicU64, + + /// cuFile reads or registrations that failed. + pub file_failures: AtomicU64, +} + +/// Per-GPU state shared with I/O threads. +struct DeviceShared { + ordinal: i32, + + /// The startup decrypt probe passed and staging holds the scratch slots. + decrypt: bool, + + slots: Arc, } /// A message to the helper thread. @@ -76,7 +204,15 @@ enum Job { fd: OwnedFd, allocation_size: u64, window: Range, - reply: oneshot::Sender>, + reply: oneshot::Sender>, + }, + /// Read ciphertext from a file straight into a staging slot. + Stage { + staged: StagedCiphertext, + fd: OwnedFd, + file_offset: u64, + length: u64, + reply: oneshot::Sender>, }, Write { id: u64, @@ -99,37 +235,61 @@ pub struct GpuHelper { next: AtomicU64, open: AtomicUsize, leaked: Arc, + shared: Arc>, + /// cuFile is worth trying: no driver-level failure has been seen. + cufile: Arc, + counters: Arc, + kernels_off: Arc>>, } impl GpuHelper { /// Resolve `mode` against hardware. Returns `None` when GPU mode is off. - pub fn start(mode: GpuMode) -> Result>> { + pub fn start(mode: GpuMode, staging_bytes: u64) -> Result>> { match mode { GpuMode::Disabled => Ok(None), GpuMode::Auto if !cfg!(feature = "cuda") => { eprintln!("racer-dataplane: stage=gpu state=disabled reason=not-built"); Ok(None) } - _ => Self::start_with(mode, Box::new(racer_gpu::Gpu::load)), + _ => Self::start_with_staging(mode, staging_bytes, Box::new(racer_gpu::Gpu::load)), } } - /// Start the helper with a custom backend loader. + /// Start the helper with a custom backend loader and the default staging cap. pub fn start_with(mode: GpuMode, loader: Loader) -> Result>> { + Self::start_with_staging(mode, crate::config::DEFAULT_GPU_STAGING_BYTES, loader) + } + + /// Start the helper with a custom backend loader and a per-GPU staging cap. + pub fn start_with_staging( + mode: GpuMode, + staging_bytes: u64, + loader: Loader, + ) -> Result>> { if mode == GpuMode::Disabled { return Ok(None); } let (jobs, receiver) = mpsc::channel(); let (ready, started) = mpsc::sync_channel(1); let leaked = Arc::new(AtomicU64::new(0)); - let helper_leaked = leaked.clone(); + let cufile = Arc::new(AtomicBool::new(true)); + let counters = Arc::new(GpuCounters::default()); + let slots = staging_bytes / SLOT_BYTES; + let kernels_off = Arc::new(Mutex::new(std::collections::HashSet::new())); + let state = HelperState { + leaked: leaked.clone(), + cufile: cufile.clone(), + counters: counters.clone(), + kernels_off: kernels_off.clone(), + slots, + }; std::thread::Builder::new() .name("racer-gpu".into()) - .spawn(move || helper_main(loader, receiver, ready, helper_leaked)) + .spawn(move || helper_main(loader, receiver, ready, state)) .map_err(|_| Error::Unavailable)?; let found = started.recv().map_err(|_| Error::Unavailable)?; - let devices = match found { - Ok(devices) if !devices.is_empty() => devices, + let (devices, decrypt) = match found { + Ok((devices, decrypt)) if !devices.is_empty() => (devices, decrypt), outcome => { let reason = match outcome { Err(error) => error.to_string(), @@ -143,9 +303,24 @@ impl GpuHelper { }; } }; + // Two scratch slots per job; the rest hold cuFile reads. + let decrypting = slots >= SCRATCH_SLOTS; + let shared = devices + .iter() + .zip(&decrypt) + .map(|(device, kernels)| DeviceShared { + ordinal: device.ordinal, + decrypt: *kernels && decrypting, + slots: Arc::new(SlotPool { + free: Mutex::new((SCRATCH_SLOTS..slots.max(SCRATCH_SLOTS)).rev().collect()), + }), + }) + .collect::>(); eprintln!( - "racer-dataplane: stage=gpu state=enabled usable={}", - devices.len() + "racer-dataplane: stage=gpu state=enabled usable={} decrypt={} staging_slots={}", + devices.len(), + shared.iter().filter(|device| device.decrypt).count(), + slots, ); Ok(Some(Arc::new(Self { jobs, @@ -153,9 +328,91 @@ impl GpuHelper { next: AtomicU64::new(1), open: AtomicUsize::new(0), leaked, + shared: Arc::new(shared), + cufile, + counters, + kernels_off, }))) } + /// GPU decryption counters. + pub fn counters(&self) -> &GpuCounters { + &self.counters + } + + fn device(&self, ordinal: i32) -> Option<&DeviceShared> { + self.shared.iter().find(|device| device.ordinal == ordinal) + } + + /// Whether sealed pages can be decrypted on this GPU. + pub fn decrypts(&self, ordinal: i32) -> bool { + self.device(ordinal).is_some_and(|device| device.decrypt) + && self + .kernels_off + .lock() + .is_ok_and(|off| !off.contains(&ordinal)) + } + + /// Free staged-ciphertext slots on this GPU. + pub fn free_slots(&self, ordinal: i32) -> usize { + self.device(ordinal) + .map_or(0, |device| device.slots.available()) + } + + /// Whether a cuFile read is worth trying. + pub fn cufile(&self) -> bool { + self.cufile.load(Ordering::Relaxed) + } + + /// Read `length` bytes at `file_offset` of `fd` into a staging slot on + /// GPU `ordinal` with cuFile. Returns [`Error::Overloaded`] when every + /// slot is taken, and [`Error::Unavailable`] when cuFile cannot read the + /// file; the caller then reads through host memory. The caller must keep + /// the bytes from changing (for example by holding a segment lease) until + /// this resolves. + pub(crate) async fn stage_file( + &self, + ordinal: i32, + fd: OwnedFd, + file_offset: u64, + length: u64, + scope: &RequestScope, + ) -> Result { + if !self.cufile() || length == 0 || length > SLOT_BYTES { + return Err(Error::Unavailable); + } + let device = self.device(ordinal).ok_or(Error::Unavailable)?; + let Some(slot) = device.slots.take() else { + self.counters.exhausted.fetch_add(1, Ordering::Relaxed); + return Err(Error::Overloaded); + }; + let staged = StagedCiphertext { + pool: device.slots.clone(), + ordinal, + slot, + }; + let (reply, mut receiver) = oneshot::channel(); + self.send(Job::Stage { + staged, + fd, + file_offset, + length, + reply, + })?; + // On cancellation the helper still runs the read; the staged slot + // then drops with the unread answer and returns to the pool. + answer(&mut receiver, scope).await? + } + + /// Where a request on GPU `ordinal` should have sealed pages delivered, + /// or None to get plaintext only. + pub fn sealed_target(self: &Arc, ordinal: i32) -> Option { + self.decrypts(ordinal).then(|| SealedTarget { + helper: self.clone(), + ordinal, + }) + } + /// Usable GPUs found at startup. pub fn devices(&self) -> &[DeviceInfo] { &self.devices @@ -220,6 +477,7 @@ impl GpuService for GpuClient { id: self.helper.next.fetch_add(1, Ordering::Relaxed), window: window.clone(), import: None, + ordinal: None, released: false, }; let (reply, answer) = oneshot::channel(); @@ -241,7 +499,9 @@ struct GpuSink { id: u64, window: Range, /// The pending import answer, until it has been checked. - import: Option>>, + import: Option>>, + /// The GPU that holds the allocation, once imported. + ordinal: Option, released: bool, } @@ -267,7 +527,7 @@ impl GpuSink { if let Some(receiver) = self.import.as_mut() { let result = answer(receiver, scope).await?; self.import = None; - result?; + self.ordinal = Some(result?); } Ok(()) } @@ -301,6 +561,7 @@ impl DirectSink for GpuSink { range: start..start + slice.length as usize, }) } + DirectPage::Sealed { page, range } => Work::Sealed(Sealed { page, range }), }; let (reply, mut receiver) = oneshot::channel(); self.helper.send(Job::Write { @@ -315,6 +576,10 @@ impl DirectSink for GpuSink { }) } + fn sealed_target(&self) -> Option { + self.helper.sealed_target(self.ordinal?) + } + fn finish<'a>(&'a mut self, scope: &'a RequestScope) -> Operation<'a, ()> { Box::pin(async move { if self.released { @@ -351,28 +616,49 @@ struct Request { import: racer_gpu::ImportedAllocation, } +/// Shared handles the helper thread starts with. +struct HelperState { + leaked: Arc, + cufile: Arc, + counters: Arc, + /// GPUs whose decrypt kernels failed after startup. + kernels_off: Arc>>, + /// Staging slots per GPU. + slots: u64, +} + /// State owned by the helper thread. struct Helper { gpu: racer_gpu::Gpu, contexts: Vec, + /// Each GPU's staging buffer, allocated on first sealed page. + staging: HashMap, + /// Registered files by (device, inode). The slab descriptors are few and + /// long lived, so one registration serves many reads. + files: HashMap<(u64, u64), racer_gpu::GpuFile>, requests: HashMap, - leaked: Arc, + state: HelperState, } +type Started = racer_gpu::Result<(Vec, Vec)>; + fn helper_main( loader: Loader, jobs: mpsc::Receiver, - ready: mpsc::SyncSender>>, - leaked: Arc, + ready: mpsc::SyncSender, + state: HelperState, ) { let mut helper = match discover(loader) { Ok((gpu, contexts, usable)) => { - let _ = ready.send(Ok(usable)); + let decrypt = contexts.iter().map(probe_decrypt).collect(); + let _ = ready.send(Ok((usable, decrypt))); Helper { gpu, contexts, + staging: HashMap::new(), + files: HashMap::new(), requests: HashMap::new(), - leaked, + state, } } Err(error) => { @@ -391,6 +677,39 @@ fn helper_main( } } +/// Queue a zero-length decrypt to learn whether the shim has working decrypt +/// kernels for this GPU. It writes nothing. +fn probe_decrypt(context: &racer_gpu::Context) -> bool { + let (Ok(staging), Ok(stream)) = (context.staging(4096), context.stream()) else { + return false; + }; + let request = racer_gpu::DecryptRequest { + key: &[0; 32], + nonce: &[0; 12], + aad: &[], + length: 0, + expected_crc: None, + }; + let queued = stream.decrypt( + &request, + &staging, + 0, + racer_gpu::Destination::Staging(&staging), + 64, + ); + let synchronized = stream.synchronize(); + let verdict = synchronized.and_then(|()| stream.verdict()); + let usable = queued.is_ok() && verdict.is_ok_and(|v| v != racer_gpu::Verdict::NotRun); + if !usable { + eprintln!( + "racer-dataplane: stage=gpu ordinal={} decrypt=unavailable reason={:?}", + context.ordinal(), + queued.err() + ); + } + usable +} + type Discovered = (racer_gpu::Gpu, Vec, Vec); /// Load the backend, log every GPU, and retain a context on each usable one. @@ -425,6 +744,25 @@ fn discover(loader: Loader) -> racer_gpu::Result { Ok((gpu, contexts, usable)) } +/// Map a GPU error on a write to a request error. +fn gpu_error(error: racer_gpu::Error) -> Error { + match error { + racer_gpu::Error::InvalidRange => Error::InvalidRange, + _ => Error::Unavailable, + } +} + +/// The (device, inode) pair that names an open file. +fn file_identity(fd: &OwnedFd) -> Result<(u64, u64)> { + use std::os::fd::AsRawFd; + // SAFETY: fstat writes only into the zeroed stat buffer. + let mut stat: libc::stat = unsafe { std::mem::zeroed() }; + if unsafe { libc::fstat(fd.as_raw_fd(), &mut stat) } != 0 { + return Err(Error::Unavailable); + } + Ok((stat.st_dev, stat.st_ino)) +} + /// Map a GPU error on import to an HTTP-visible error. fn import_error(error: racer_gpu::Error) -> Error { match error { @@ -451,6 +789,18 @@ impl Helper { self.note_leaks(); let _ = reply.send(result); } + Job::Stage { + staged, + fd, + file_offset, + length, + reply, + } => { + let result = self.stage(&staged, fd, file_offset, length); + self.note_leaks(); + // A dropped receiver drops the slot, which frees it. + let _ = reply.send(result.map(|()| staged)); + } Job::Write { id, offset, @@ -475,7 +825,7 @@ impl Helper { } /// Try each usable GPU until one owns the allocation. - fn import(&mut self, id: u64, fd: OwnedFd, size: u64, window: Range) -> Result<()> { + fn import(&mut self, id: u64, fd: OwnedFd, size: u64, window: Range) -> Result { let mut last = racer_gpu::Error::WrongDevice; for context in &self.contexts { let attempt = fd.try_clone().map_err(|_| Error::Unavailable)?; @@ -483,7 +833,7 @@ impl Helper { Ok(import) => { let stream = context.stream().map_err(import_error)?; self.requests.insert(id, Request { stream, import }); - return Ok(()); + return Ok(context.ordinal()); } Err(racer_gpu::Error::WrongDevice) => continue, Err(error) => { @@ -496,21 +846,186 @@ impl Helper { } fn write(&mut self, id: u64, offset: u64, work: &Work) -> Result<()> { + let plaintext = match work { + Work::Plaintext(plaintext) => plaintext, + Work::Sealed(sealed) => return self.write_sealed(id, offset, sealed), + }; let request = self.requests.get(&id).ok_or(Error::Internal)?; let destination = racer_gpu::Destination::Import(&request.import); - let queued = match work { - Work::Plaintext(plaintext) => { - request - .stream - .copy_from_host(destination, offset, plaintext.bytes()) - } - }; + let queued = request + .stream + .copy_from_host(destination, offset, plaintext.bytes()); // Synchronize even after a failed queue call: part of it may run. let synchronized = request.stream.synchronize(); - queued.and(synchronized).map_err(|error| match error { - racer_gpu::Error::InvalidRange => Error::InvalidRange, - _ => Error::Unavailable, - }) + queued.and(synchronized).map_err(gpu_error) + } + + /// This GPU's staging buffer, allocated on first use. + fn staging(&mut self, ordinal: i32) -> Result<&racer_gpu::StagingBuffer> { + if !self.staging.contains_key(&ordinal) { + let context = self + .contexts + .iter() + .find(|context| context.ordinal() == ordinal) + .ok_or(Error::Internal)?; + let size = self.state.slots.max(SCRATCH_SLOTS) * SLOT_BYTES; + let buffer = context.staging(size).map_err(|error| { + eprintln!( + "racer-dataplane: stage=gpu ordinal={ordinal} state=staging-failed reason={error}" + ); + Error::Overloaded + })?; + self.staging.insert(ordinal, buffer); + } + self.staging.get(&ordinal).ok_or(Error::Internal) + } + + /// Read a file range into a staged slot with cuFile. + fn stage( + &mut self, + staged: &StagedCiphertext, + fd: OwnedFd, + file_offset: u64, + length: u64, + ) -> Result<()> { + let identity = file_identity(&fd)?; + self.staging(staged.ordinal)?; + let staging = self.staging.get(&staged.ordinal).ok_or(Error::Internal)?; + let counters = &self.state.counters; + if let Err(error) = staging.register_cufile() { + counters.file_failures.fetch_add(1, Ordering::Relaxed); + self.disable_cufile(error); + return Err(Error::Unavailable); + } + if !self.files.contains_key(&identity) { + if self.files.len() >= MAX_FILES { + self.files.clear(); + } + match self.gpu.open_file(fd) { + Ok(file) => { + self.files.insert(identity, file); + } + Err(error) => { + counters.file_failures.fetch_add(1, Ordering::Relaxed); + self.disable_cufile(error); + return Err(Error::Unavailable); + } + } + } + let file = self.files.get(&identity).ok_or(Error::Internal)?; + let staging = self.staging.get(&staged.ordinal).ok_or(Error::Internal)?; + match file.read(staging, staged.offset(), file_offset, length) { + Ok(()) => { + counters.staged.fetch_add(1, Ordering::Relaxed); + Ok(()) + } + Err(error) => { + counters.file_failures.fetch_add(1, Ordering::Relaxed); + // A failed handle may be stale; register again next time. + self.files.remove(&identity); + eprintln!("racer-dataplane: stage=gpu state=cufile-read-failed reason={error}"); + Err(Error::Unavailable) + } + } + } + + /// Stop trying cuFile after a failure that is not about one read. + fn disable_cufile(&self, error: racer_gpu::Error) { + if self.state.cufile.swap(false, Ordering::Relaxed) { + eprintln!("racer-dataplane: stage=gpu state=cufile-disabled reason={error}"); + } + } + + /// Verify and decrypt a sealed page, then write only the requested range. + fn write_sealed(&mut self, id: u64, offset: u64, sealed: &Sealed) -> Result<()> { + let ordinal = self + .requests + .get(&id) + .ok_or(Error::Internal)? + .import + .ordinal(); + self.staging(ordinal)?; + let staging = self.staging.get(&ordinal).ok_or(Error::Internal)?; + let request = self.requests.get(&id).ok_or(Error::Internal)?; + let page = &sealed.page; + let length = u64::from(page.plaintext_length()); + if sealed.range.start >= sealed.range.end || sealed.range.end > length { + return Err(Error::InvalidRange); + } + let stream = &request.stream; + let source = match &page.source { + SealedSource::Host(copy) => { + let bytes = copy.copy.ciphertext.bytes(); + if bytes.len() as u64 != length + 16 { + return Err(Error::CorruptRecord); + } + let copied = stream.copy_from_host( + racer_gpu::Destination::Staging(staging), + CIPHERTEXT_SCRATCH, + bytes, + ); + if let Err(error) = copied { + let _ = stream.synchronize(); + return Err(gpu_error(error)); + } + CIPHERTEXT_SCRATCH + } + SealedSource::Staged { staged, .. } => { + if staged.ordinal() != ordinal { + return Err(Error::Internal); + } + staged.offset() + } + }; + let request_key = racer_gpu::DecryptRequest { + key: page.key.key(), + nonce: page.key.nonce(), + aad: &page.aad, + length, + expected_crc: page.expected_crc, + }; + // A whole page decrypts straight into the import. Part of a page + // decrypts into scratch first, because the tag covers the whole page + // and nothing outside the window may be written. + let whole = sealed.range.start == 0 && sealed.range.end == length; + let (destination, at) = if whole { + (racer_gpu::Destination::Import(&request.import), offset) + } else { + (racer_gpu::Destination::Staging(staging), PLAINTEXT_SCRATCH) + }; + let queued = stream.decrypt(&request_key, staging, source, destination, at); + let synchronized = stream.synchronize(); + if let Err(racer_gpu::Error::KernelUnavailable) = queued { + if let Ok(mut off) = self.state.kernels_off.lock() { + off.insert(ordinal); + } + return Err(Error::Unavailable); + } + queued.and(synchronized).map_err(gpu_error)?; + match stream.verdict().map_err(gpu_error)? { + racer_gpu::Verdict::Passed { .. } => {} + racer_gpu::Verdict::CrcMismatch { .. } | racer_gpu::Verdict::TagMismatch { .. } => { + self.state.counters.rejected.fetch_add(1, Ordering::Relaxed); + return Err(Error::CorruptRecord); + } + racer_gpu::Verdict::NotRun => return Err(Error::Unavailable), + } + if !whole { + let copied = stream.copy_from_staging( + racer_gpu::Destination::Import(&request.import), + offset, + staging, + PLAINTEXT_SCRATCH + sealed.range.start, + sealed.range.end - sealed.range.start, + ); + let synchronized = stream.synchronize(); + copied.and(synchronized).map_err(gpu_error)?; + } + self.state + .counters + .decrypted + .fetch_add(1, Ordering::Relaxed); + Ok(()) } /// Drop the stream (which synchronizes it), then the import. @@ -532,7 +1047,7 @@ impl Helper { fn note_leaks(&self) { let leaked = self.gpu.leaked(); - if self.leaked.swap(leaked, Ordering::Relaxed) != leaked { + if self.state.leaked.swap(leaked, Ordering::Relaxed) != leaked { eprintln!("racer-dataplane: stage=gpu state=leaked total={leaked}"); } } diff --git a/cmd/racer-dataplane/src/read.rs b/cmd/racer-dataplane/src/read.rs index 7285674ed..3913908e9 100644 --- a/cmd/racer-dataplane/src/read.rs +++ b/cmd/racer-dataplane/src/read.rs @@ -37,6 +37,7 @@ pub mod fill; pub mod flight; pub mod metadata; pub mod range_stream; +pub mod sealed; pub mod warm; /// Metadata and its optional stream, with no independent range to disagree with it. diff --git a/cmd/racer-dataplane/src/read/dispatch.rs b/cmd/racer-dataplane/src/read/dispatch.rs index a7be805b9..8de785af8 100644 --- a/cmd/racer-dataplane/src/read/dispatch.rs +++ b/cmd/racer-dataplane/src/read/dispatch.rs @@ -117,6 +117,16 @@ enum Work { std::sync::Arc, PeerOriginContext, ), + /// One read window page for direct GPU delivery: like `Fetch`, but + /// ciphertext the GPU can verify is returned sealed. + FetchSealed( + PageId, + std::sync::Arc, + PeerOriginContext, + super::sealed::SealedTarget, + ), + /// The GPU's verdict on a sealed page this worker supplied. + SealedVerdict(super::sealed::SealedReport, bool), Publish(VersionMetadata), Retained(ObjectVersion), Peer(VerifiedRequest, std::sync::Arc), @@ -125,6 +135,7 @@ enum Value { Held(bool, std::time::SystemTime, HoldEpoch), Metadata(ObjectMetadata), Page(PageResult), + Fetched(super::sealed::Fetched), Cached(Option), Published, Retained(Option), @@ -873,6 +884,72 @@ impl WorkerDirectory { Ok((value, budget)) })) } + /// Like [`Self::start_page`], but ciphertext the GPU behind `target` + /// can verify comes back sealed instead of decrypted on the CPU. + pub(crate) fn start_page_sealed( + &self, + page: PageId, + membership: std::sync::Arc, + context: &OriginContext, + scope: &RequestScope, + budget: AcquisitionBudget, + target: super::sealed::SealedTarget, + ) -> Result, AcquisitionBudget)>> { + if page.version.object != context.object { + return Err(Error::InvalidRequest); + } + let owner = self.page_owner(&page)?; + let receipt = self.submit( + owner, + Work::FetchSealed(page, membership, self.seal(context, scope)?, target), + scope, + Some(budget), + )?; + Ok(Box::pin(async move { + let completion = receipt.await?; + let budget = completion.budget.ok_or(Error::StaleFlight)?; + let value = completion.value.and_then(|value| match value { + Value::Fetched(value) => Ok(value), + _ => Err(Error::StaleFlight), + }); + Ok((value, budget)) + })) + } + + /// Tell a sealed page's owner what the GPU decided, in the background. + pub(crate) fn report_sealed( + &self, + report: super::sealed::SealedReport, + passed: bool, + scope: &RequestScope, + ) { + let Ok(owner) = self.page_owner(&report.page) else { + return; + }; + if self.is_local(owner) { + if let Ok(local) = self.local() { + local.fill.sealed_verdict(report, passed); + } + return; + } + let Ok(permit) = uring_runtime::drivers::reserve() else { + return; + }; + // The request may already be ending; the verdict must still arrive. + let deadline = uring_runtime::environment::now() + std::time::Duration::from_secs(5); + let Ok(detached) = RequestScope::new(scope.request, deadline) else { + return; + }; + if let Ok(receipt) = + self.submit(owner, Work::SealedVerdict(report, passed), &detached, None) + { + permit.submit_detached(Box::pin(async move { + let _ = receipt.await; + Ok::<(), Error>(()) + })); + } + } + pub fn publish_metadata<'a>( &'a self, metadata: VersionMetadata, @@ -1229,6 +1306,30 @@ async fn execute( .await .map(Value::Page) } + Work::FetchSealed(page, membership, envelope, target) => { + if let Warmed::Local(result) = + local.streams.directory().warmed_page(&page, scope).await? + { + return Ok(Value::Fetched(super::sealed::Fetched::Page(*result))); + } + let context = local.open_context(envelope)?; + local + .fill + .acquire_sealed( + page, + membership, + &context, + scope, + budget.ok_or(Error::StaleFlight)?, + &target, + ) + .await + .map(Value::Fetched) + } + Work::SealedVerdict(report, passed) => { + local.fill.sealed_verdict(report, passed); + Ok(Value::Published) + } Work::Publish(metadata) => { local.metadata.publish_version(metadata)?; Ok(Value::Published) diff --git a/cmd/racer-dataplane/src/read/fill.rs b/cmd/racer-dataplane/src/read/fill.rs index 267ae5dac..038224b32 100644 --- a/cmd/racer-dataplane/src/read/fill.rs +++ b/cmd/racer-dataplane/src/read/fill.rs @@ -77,6 +77,18 @@ type LocalPlaintextKeys = std::cell::RefCell< std::collections::BTreeMap>>>, >; +/// A host descriptor to hand to cuFile, or None for simulated descriptors. +fn host_descriptor( + descriptor: &uring_runtime::reactor::descriptor::Descriptor, +) -> Option { + use std::os::fd::AsFd; + #[cfg(test)] + if descriptor.as_sim().is_some() { + return None; + } + descriptor.as_fd().try_clone_to_owned().ok() +} + /// Optional writes may use half of the aggregate class budget, with a floor /// sufficient for one valid page when the configured budget can hold it. Owned /// usage counts against this conservative ceiling; quota admission remains the @@ -889,6 +901,195 @@ impl Fill { } } + /// Acquire one page for direct GPU delivery: verified plaintext when the + /// CPU has it anyway, otherwise ciphertext for the GPU to verify and + /// decrypt. Network and disk copies keep the usual single-flight sharing, + /// memory publication and disk persistence; only the CPU decrypt and the + /// plaintext publication are skipped. + pub(crate) async fn acquire_sealed( + &self, + page: PageId, + membership: Arc, + context: &OriginContext, + scope: &RequestScope, + budget: &mut AcquisitionBudget, + target: &super::sealed::SealedTarget, + ) -> Result { + use super::sealed::Fetched; + // Verified memory, and pages another reader already authenticates, + // keep the plaintext path: the bytes are on the CPU anyway. + if self.cached_page(&page, scope)?.is_some() { + return self + .acquire(page, membership, context, scope, budget) + .await + .map(Fetched::Page); + } + if target.helper.cufile() + && self + .dependencies + .writer + .copy_only(&page) + .ok() + .flatten() + .is_none() + { + match self.stage_from_disk(&page, scope, target).await { + Ok(Some(sealed)) => return Ok(Fetched::Sealed(sealed)), + Ok(None) => {} + Err(error @ (Error::Cancelled | Error::DeadlineExceeded)) => return Err(error), + // Any other failure reads the page through host memory. + Err(_) => {} + } + } + let acquired = self + .acquire_with_prefetch( + page.clone(), + membership.clone(), + context, + scope, + budget, + None, + false, + true, + ) + .await?; + match acquired { + AcquiredPage::Plaintext(result) => Ok(Fetched::Page(result)), + AcquiredPage::Ciphertext(copy) => match self.seal(&page, copy) { + Ok(sealed) => Ok(Fetched::Sealed(sealed)), + // A key the GPU cannot use, or an AAD too long for it: the + // CPU authenticates the page as for a Get. + Err(_) => self + .acquire(page, membership, context, scope, budget) + .await + .map(Fetched::Page), + }, + } + } + + /// Package host ciphertext for GPU verification. + fn seal(&self, page: &PageId, copy: UnverifiedPage) -> Result> { + validate_copy(©.copy, page)?; + let envelope = copy.copy.ciphertext.envelope().clone(); + let sealed = Box::new(super::sealed::SealedPage { + metadata: copy.copy.metadata.clone(), + aad: crate::security::page_aad(&envelope)?, + key: self.dependencies.crypto.derive_page_key(&envelope)?, + expected_crc: copy.copy.ciphertext.cached_checksum(), + envelope, + source: super::sealed::SealedSource::Host(copy), + }); + sealed.validate_for(page)?; + Ok(sealed) + } + + /// Read a local v2 record's ciphertext straight into GPU staging with + /// cuFile. Only the header passes through host memory. None means the + /// page must be read through host memory: a miss, a v1 or unaligned + /// record, no free staging slot, or cuFile unavailable. + async fn stage_from_disk( + &self, + page: &PageId, + scope: &RequestScope, + target: &super::sealed::SealedTarget, + ) -> Result>> { + if target.helper.free_slots(target.ordinal) == 0 { + target + .helper + .counters() + .exhausted + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + return Ok(None); + } + let Some(record) = self + .dependencies + .disk + .direct_read(page, scope, |amount| { + self.reserve_with_reclamation( + &page.version.object.cache, + ResourceClass::Ciphertext, + amount, + ) + }) + .await? + else { + return Ok(None); + }; + if !record.ciphertext.aligned { + return Ok(None); + } + let Some(fd) = host_descriptor(&record.ciphertext.descriptor) else { + return Ok(None); + }; + let envelope = record.envelope.clone(); + if record.ciphertext.length != envelope.ciphertext_length as usize { + return Ok(None); + } + let aad = crate::security::page_aad(&envelope)?; + let key = self.dependencies.crypto.derive_page_key(&envelope)?; + let staged = target + .helper + .stage_file( + target.ordinal, + fd, + record.ciphertext.offset, + record.read_length as u64, + scope, + ) + .await; + // The read finished (or was abandoned): the bytes may now be recycled. + let crate::store::DirectRecord { + token, + metadata, + checksum, + lease, + .. + } = record; + drop(lease); + let staged = match staged { + Ok(staged) => staged, + Err(error @ (Error::Cancelled | Error::DeadlineExceeded)) => return Err(error), + Err(_) => return Ok(None), + }; + let sealed = Box::new(super::sealed::SealedPage { + metadata, + envelope, + aad, + key, + expected_crc: Some(checksum), + source: super::sealed::SealedSource::Staged { staged, token }, + }); + sealed.validate_for(page)?; + self.metrics.record(Event::DiskHit, 1); + Ok(Some(sealed)) + } + + /// Act on the GPU's verdict for a sealed page this worker supplied: a + /// rejected copy is dropped from memory and disk, as a failed CPU decrypt + /// would drop it; an authenticated network copy may now be persisted. + pub(crate) fn sealed_verdict(&self, report: super::sealed::SealedReport, passed: bool) { + let super::sealed::SealedReport { page, copy, token } = report; + if passed { + if let Some(copy) = copy + && copy.disk_token.is_none() + { + let observation = self.dependencies.writer.retention().owned_only(&page); + self.persist_copy(copy.copy, observation); + } + return; + } + self.metrics.record(Event::CorruptMiss, 1); + if let Some(copy) = © { + self.dependencies.memory.invalidate_ciphertext(copy); + if let Some(token) = ©.disk_token { + let _ = self.dependencies.disk.invalidate(token); + } + } + if let Some(token) = &token { + let _ = self.dependencies.disk.invalidate(token); + } + } + /// Warm gates own transfer exclusion until accepted acquisition work is fenced. pub(crate) async fn acquire_warm( &self, @@ -2113,7 +2314,20 @@ impl Fill { if !observation.eligible() || result.validate_metadata().is_err() { return; } - let id = result.plaintext.page(); + self.persist_copy(result.copy(), observation); + } + + /// Queue an authenticated copy for disk under the reader's latched + /// retention decision. The caller must have authenticated the bytes. + fn persist_copy( + &self, + copy: crate::memory::CiphertextCopy, + observation: crate::retention::Observation, + ) { + if !observation.eligible() || copy.validate_metadata().is_err() { + return; + } + let id = ©.ciphertext.envelope().page.clone(); if self .dependencies .writer @@ -2135,13 +2349,13 @@ impl Fill { { return; } - let cache = &result.metadata.version.object.cache; + let cache = ©.metadata.version.object.cache.clone(); let retention = self.dependencies.writer.retention(); // Ownership is current here, but second-sight eligibility remains the // latched observation. Reclassification cannot self-promote Bloom state. let owned = retention.owned_only(id).eligible(); let admission = &self.dependencies.admission; - let amount = result.ciphertext.bytes().len(); + let amount = copy.ciphertext.bytes().len(); // Optional pages of an over-share cache do not displace other caches' // data once free segments reach the reserve. Owned pages compete // through eviction instead. @@ -2166,7 +2380,7 @@ impl Fill { .reserve(Some(cache), ResourceClass::DirtyCiphertext, amount) .map_err(Error::from) }; - let page_charge = result + let page_charge = copy .ciphertext .inner .reservation() @@ -2187,7 +2401,7 @@ impl Fill { let result = self .dependencies .writer - .enqueue_reclaiming(result.copy(), dirty, |amount| { + .enqueue_reclaiming(copy, dirty, |amount| { // Ciphertext owns both the immutable page and its disk staging. // The one-page floor includes BOTH, even on tiny slabs. The page // term is the page's own Ciphertext charge, which for aligned diff --git a/cmd/racer-dataplane/src/read/range_stream.rs b/cmd/racer-dataplane/src/read/range_stream.rs index d0431e5bb..bb7f88fea 100644 --- a/cmd/racer-dataplane/src/read/range_stream.rs +++ b/cmd/racer-dataplane/src/read/range_stream.rs @@ -23,6 +23,10 @@ use crate::model::PageNumber; use crate::model::ResolvedRange; #[cfg(test)] use crate::read::dispatch::WorkerMap; +use crate::read::sealed::Fetched; +use crate::read::sealed::SealedPage; +use crate::read::sealed::SealedReport; +use crate::read::sealed::SealedTarget; use crate::runtime::RequestScope; use crate::security::OriginContext; use crate::store::holds::HoldEpoch; @@ -478,6 +482,16 @@ impl RangeBudget { type Fetch = Operation<'static, (Result, AcquisitionBudget)>; +/// A direct fill in sealed mode, which may return ciphertext for the GPU. +type SealedFetch = Operation<'static, (Result, AcquisitionBudget)>; + +/// One page for a direct read: plaintext, or ciphertext the sink verifies. +pub(crate) enum DirectItem { + Plaintext(ReaderLease), + /// A sealed page and the slice of it the range covers. + Sealed(Box, crate::model::PageSlice), +} + /// One page of the read window, `slots[i]` is page `next + i`. #[allow(clippy::large_enum_variant)] // Avoid allocating again when an in-window page becomes ready. enum Slot { @@ -488,6 +502,10 @@ enum Slot { Missing { select: bool }, /// A direct fill through the owning worker. Fetching(Fetch), + /// A direct fill in sealed mode. + FetchingSealed(SealedFetch), + /// Fetched as ciphertext for the GPU to verify. + Sealed(Box), /// Covered by an outstanding provider selection. Selecting, /// Fetched, or failed. An error waits here until the page reaches the head. @@ -576,6 +594,9 @@ pub struct RangeStream { spill: Option, + /// Set for direct GPU reads: misses are fetched sealed for this GPU. + sealed: Option, + terminated: bool, } @@ -640,6 +661,7 @@ impl RangeStreams { slots: VecDeque::new(), selections: Vec::new(), spill: None, + sealed: None, terminated: false, }) } @@ -678,6 +700,21 @@ impl RangeStream { Ok(()) } + /// Fetch pages that are not verified plaintext in memory as sealed pages + /// for `target`'s GPU. Call before the first delivery. + pub(crate) fn set_sealed(&mut self, target: SealedTarget) -> Result<()> { + if self.terminated || !self.slots.is_empty() { + return Err(Error::InvalidRequest); + } + self.sealed = Some(target); + Ok(()) + } + + /// Tell a sealed page's owner what the GPU decided. + pub(crate) fn report_sealed(&self, report: SealedReport, passed: bool) { + self.directory.report_sealed(report, passed, &self.scope); + } + /// Release exactly one delivered page and its retained plaintext charge. pub fn release_page(&mut self, number: PageNumber, length: u32) -> Result<()> { self.credits @@ -739,6 +776,7 @@ impl RangeStream { }; let directory = self.directory.clone(); let scope = scope.clone(); + let sealed = self.sealed.is_some(); let spilled = self .spill .as_mut() @@ -755,6 +793,15 @@ impl RangeStream { .insert(number.0, Spilling::Holding(holding)); Box::pin(async move { directory.local_page(page, &scope).await }) } + // A sealed read probes memory only: a spilled page on disk is + // then fetched sealed, so the GPU decrypts it. + Some(Spilling::Spilled) if sealed => { + Box::pin(async move { directory.cached_page(page, &scope).await }) + } + Some(Spilling::Warming(warming)) if sealed => Box::pin(async move { + let _ = warming.await; + directory.cached_page(page, &scope).await + }), // A spilled page is read back from memory or its held disk copy. // If that fails the page is fetched normally. Some(Spilling::Spilled) => { @@ -828,6 +875,28 @@ impl RangeStream { } Poll::Ready(Err(error)) => Slot::Ready(Err(error)), }, + Slot::FetchingSealed(fetch) => match fetch.as_mut().poll(cx) { + Poll::Pending => continue, + Poll::Ready(Ok((result, remaining))) => { + let completed = self.budget.complete(remaining); + match result { + Ok(Fetched::Page(page)) => { + if let Some(lease) = &self.lease { + lease.offer(&page); + } + Slot::Ready(completed.map(|()| page)) + } + Ok(Fetched::Sealed(sealed)) => match completed + .and_then(|()| self.validate_sealed(&sealed, number)) + { + Ok(()) => Slot::Sealed(sealed), + Err(error) => Slot::Ready(Err(error)), + }, + Err(error) => Slot::Ready(completed.and(Err(error))), + } + } + Poll::Ready(Err(error)) => Slot::Ready(Err(error)), + }, _ => continue, }; self.slots[index] = next; @@ -903,7 +972,7 @@ impl RangeStream { || self .slots .iter() - .any(|slot| matches!(slot, Slot::Fetching(_))) + .any(|slot| matches!(slot, Slot::Fetching(_) | Slot::FetchingSealed(_))) } /// Start a direct fill for the head page, up to `DIRECT_AHEAD` more holes, @@ -933,7 +1002,7 @@ impl RangeStream { .slots .iter() .skip(1) - .filter(|slot| matches!(slot, Slot::Fetching(_))) + .filter(|slot| matches!(slot, Slot::Fetching(_) | Slot::FetchingSealed(_))) .count(); // Pages another window is selecting or filling arrive through its // fan-out. Only the head is always filled directly, so it never waits @@ -972,20 +1041,34 @@ impl RangeStream { }; let mut child_scope = scope.clone(); child_scope.deadline.0 = child.deadline(); - let fetch = self.directory.start_page( - PageId { - version: self.metadata.version.clone(), - number: PageNumber(self.page_at(index)), - }, - self.membership.clone(), - &self.context, - &child_scope, - child, - ); - self.slots[index] = match fetch { - Ok(fetch) => Slot::Fetching(fetch), - Err(error) => Slot::Ready(Err(error)), + let page = PageId { + version: self.metadata.version.clone(), + number: PageNumber(self.page_at(index)), }; + let fetch = match &self.sealed { + Some(target) => self + .directory + .start_page_sealed( + page, + self.membership.clone(), + &self.context, + &child_scope, + child, + target.clone(), + ) + .map(Slot::FetchingSealed), + None => self + .directory + .start_page( + page, + self.membership.clone(), + &self.context, + &child_scope, + child, + ) + .map(Slot::Fetching), + }; + self.slots[index] = fetch.unwrap_or_else(|error| Slot::Ready(Err(error))); Ok(true) } @@ -1139,7 +1222,7 @@ impl RangeStream { wants.insert(number); busy.insert(number); } - Slot::Fetching(_) => { + Slot::Fetching(_) | Slot::FetchingSealed(_) => { busy.insert(number); } Slot::Probing(_) | Slot::Missing { .. } => { @@ -1337,6 +1420,13 @@ impl RangeStream { } scope } + fn validate_sealed(&self, sealed: &SealedPage, number: PageNumber) -> Result<()> { + validate_pin(&self.metadata, &sealed.metadata)?; + sealed.validate_for(&PageId { + version: self.metadata.version.clone(), + number, + }) + } fn validate(&self, result: &PageResult, number: PageNumber) -> Result<()> { validate_pin(&self.metadata, &result.metadata)?; result.validate_for(&PageId { @@ -1347,6 +1437,22 @@ impl RangeStream { /// Deliver the next page in order. Pending futures live in the stream, so /// dropping next_slice cannot restart a page or refill its retries. pub fn next_slice(&mut self) -> Operation<'_, Option> { + Box::pin(async move { + match self.next_direct().await? { + None => Ok(None), + Some(DirectItem::Plaintext(reader)) => Ok(Some(reader)), + // Only direct GPU reads set a sealed target. + Some(DirectItem::Sealed(..)) => { + self.terminate(); + Err(Error::Internal) + } + } + }) + } + + /// Deliver the next page in order for a direct read, sealed if the + /// stream has a sealed target and the page was fetched as ciphertext. + pub(crate) fn next_direct(&mut self) -> Operation<'_, Option> { Box::pin(async move { if self.terminated { return Ok(None); @@ -1384,10 +1490,10 @@ impl RangeStream { /// Whether the head page can be handed to the client now. fn deliverable(&self) -> bool { - matches!(self.slots.front(), Some(Slot::Ready(_))) + matches!(self.slots.front(), Some(Slot::Ready(_) | Slot::Sealed(_))) } - async fn deliver(&mut self) -> Result> { + async fn deliver(&mut self) -> Result> { let scope = self.operation_scope(); let cancellation = scope.cancellation.subscribe()?; poll_fn(|cx| -> Poll> { @@ -1408,20 +1514,37 @@ impl RangeStream { return Ok(None); } scope.check()?; - let Some(Slot::Ready(result)) = self.slots.pop_front() else { - return Err(Error::StaleFlight); - }; let number = PageNumber(self.next); - self.next += 1; - let result = result?; + let result = match self.slots.pop_front() { + Some(Slot::Ready(result)) => { + self.next += 1; + result? + } + Some(Slot::Sealed(sealed)) => { + self.next += 1; + self.validate_sealed(&sealed, number)?; + let slice = self.range.slice_at(number)?.ok_or(Error::CorruptRecord)?; + self.issue(number)?; + return Ok(Some(DirectItem::Sealed(sealed, slice))); + } + _ => return Err(Error::StaleFlight), + }; self.validate(&result, number)?; let slice = self.range.slice_at(number)?.ok_or(Error::CorruptRecord)?; + self.issue(number)?; + self.retained.insert(number, result.plaintext.clone()); + self.delivery + .attach(result.plaintext, slice) + .map(|reader| Some(DirectItem::Plaintext(reader))) + } + + /// Account one page as handed to the client, and drop its spill hold. + fn issue(&mut self, number: PageNumber) -> Result<()> { self.credits .as_mut() .ok_or(Error::InvalidRequest)? .issued(number) .map_err(window_error)?; - self.retained.insert(number, result.plaintext.clone()); if let Some(spill) = self.spill.as_mut() && let Some(epoch) = spill.held.remove(&number.0) { @@ -1431,14 +1554,23 @@ impl RangeStream { self.scope.request, ); } - self.delivery.attach(result.plaintext, slice).map(Some) + Ok(()) } /// Pages admitted into the window: fetched, being fetched or selected. pub fn buffered_pages(&self) -> usize { self.slots .iter() - .filter(|slot| matches!(slot, Slot::Fetching(_) | Slot::Selecting | Slot::Ready(_))) + .filter(|slot| { + matches!( + slot, + Slot::Fetching(_) + | Slot::FetchingSealed(_) + | Slot::Selecting + | Slot::Ready(_) + | Slot::Sealed(_) + ) + }) .count() } } diff --git a/cmd/racer-dataplane/src/read/sealed.rs b/cmd/racer-dataplane/src/read/sealed.rs new file mode 100644 index 000000000..a00127495 --- /dev/null +++ b/cmd/racer-dataplane/src/read/sealed.rs @@ -0,0 +1,206 @@ +//! Sealed pages: ciphertext a GPU authenticates and decrypts itself. +//! +//! A `gpu-read` asks its range stream for sealed pages +//! ([`super::range_stream::RangeStream::set_sealed`]). Pages that are verified +//! plaintext in memory still arrive as [`Fetched::Page`], and so do pages that +//! the CPU had to produce anyway (origin fills, which encrypt) or that another +//! reader authenticated first. Every other page arrives as a [`SealedPage`]: +//! the envelope, the AAD, the expected CRC, the XAES-derived page key, and the +//! ciphertext in host memory or already in GPU staging. +//! +//! The derived key is wiped on drop and is never logged: [`SealedPage`] has no +//! `Debug` implementation. + +use crate::error::Error; +use crate::error::Result; +use crate::gpu::GpuHelper; +use crate::gpu::StagedCiphertext; +use crate::memory::PageResult; +use crate::memory::UnverifiedPage; +use crate::model::ObjectMetadata; +use crate::model::PageEnvelope; +use crate::model::PageId; +use crate::store::ReadToken; +use std::sync::Arc; + +/// Where a GPU should read a sealed page from. +#[derive(Clone)] +pub struct SealedTarget { + pub(crate) helper: Arc, + + /// The GPU that holds the request's allocation. + pub(crate) ordinal: i32, +} + +impl SealedTarget { + /// The GPU that holds the request's allocation. + pub fn ordinal(&self) -> i32 { + self.ordinal + } +} + +/// One fetched window page in sealed mode. +pub enum Fetched { + /// Verified plaintext. + Page(PageResult), + + /// Ciphertext for the GPU to verify and decrypt. + Sealed(Box), +} + +/// Where a sealed page's ciphertext is. +pub enum SealedSource { + /// Ciphertext and tag in host memory: retained, pending, read from disk + /// through the slab, or received from a peer. + Host(UnverifiedPage), + + /// Ciphertext already in the target GPU's staging memory, read from a + /// local v2 disk record with cuFile. + Staged { + staged: StagedCiphertext, + + /// The disk mapping the bytes came from. + token: ReadToken, + }, +} + +/// A page that only the GPU will authenticate. Nothing here is proof of +/// authenticity: the GPU checks the CRC and the AEAD tag before it writes. +pub struct SealedPage { + pub metadata: ObjectMetadata, + + pub envelope: PageEnvelope, + + /// `security::page_aad(&envelope)`. + pub aad: Vec, + + /// The XAES-derived AES-256-GCM key and nonce. Wiped on drop. + pub key: racer_crypto::DerivedPageKey, + + /// The stored CRC-64 of ciphertext and tag, when known. + pub expected_crc: Option, + + pub source: SealedSource, +} + +/// What a page's owning worker needs to act on the GPU's verdict. +pub struct SealedReport { + pub(crate) page: PageId, + + /// The host copy, to drop from memory on failure or persist on success. + pub(crate) copy: Option, + + /// The disk mapping a staged copy came from, to drop on failure. + pub(crate) token: Option, +} + +impl SealedPage { + /// Owner-side facts about where the bytes came from. Cheap: it shares + /// the ciphertext and copies no secrets. + pub fn report(&self) -> SealedReport { + let (copy, token) = match &self.source { + SealedSource::Host(copy) => (Some(copy.clone()), None), + SealedSource::Staged { token, .. } => (None, Some(token.clone())), + }; + SealedReport { + page: self.page().clone(), + copy, + token, + } + } + + /// The page identity from the envelope. + pub fn page(&self) -> &PageId { + &self.envelope.page + } + + /// Plaintext bytes in the whole page. + pub fn plaintext_length(&self) -> u32 { + self.envelope.plaintext_length + } + + /// Structural checks only: the identity, lengths and AAD bound agree. + pub fn validate_for(&self, page: &PageId) -> Result<()> { + self.metadata.validate()?; + self.envelope.validate()?; + if self.page() != page + || self.metadata.version != page.version + || self.metadata.immutable().page_length(page)? != self.envelope.plaintext_length + || self.aad.len() > racer_gpu::MAX_AAD + { + return Err(Error::CorruptRecord); + } + if let SealedSource::Host(copy) = &self.source { + let ciphertext = ©.copy.ciphertext; + if ciphertext.envelope() != &self.envelope + || ciphertext.bytes().len() != self.envelope.ciphertext_length as usize + { + return Err(Error::CorruptRecord); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::memory::CiphertextCopy; + + fn sealed(result: &PageResult) -> SealedPage { + let envelope = result.ciphertext.envelope().clone(); + SealedPage { + metadata: result.metadata.clone(), + aad: crate::security::page_aad(&envelope).unwrap(), + key: racer_crypto::xaes_derive(&[7; 32], &envelope.nonce.0), + expected_crc: Some(result.ciphertext.checksum()), + envelope, + source: SealedSource::Host(UnverifiedPage { + copy: CiphertextCopy { + metadata: result.metadata.clone(), + ciphertext: result.ciphertext.clone(), + }, + disk_token: None, + }), + } + } + + #[test] + fn validate_checks_identity_lengths_and_aad_bound() { + use crate::model::*; + let admission = flow_control::Quotas::new(crate::admission::AdmissionPolicy::new( + crate::test_support::cluster::config(false).limits, + )); + let metadata = ObjectMetadata { + content_type: None, + version: ObjectVersion { + object: ObjectId { + cache: racer_control_wire::CacheId( + "00000000-0000-4000-8000-000000000001".into(), + ), + key: CacheKey([0; 32]), + }, + etag: StrongEtag::test_value("v1"), + }, + length: 37, + expires_at: ExpiresAt::from_system_time(std::time::UNIX_EPOCH).unwrap(), + }; + let result = crate::read::range_stream::tests::page_result(&admission, &metadata, 0); + let page = result.plaintext.page().clone(); + let mut sealed = sealed(&result); + sealed.validate_for(&page).unwrap(); + assert_eq!(sealed.plaintext_length(), 37); + + let mut other = page.clone(); + other.number = crate::model::PageNumber(1); + assert_eq!(sealed.validate_for(&other), Err(Error::CorruptRecord)); + + sealed.aad = vec![0; racer_gpu::MAX_AAD + 1]; + assert_eq!(sealed.validate_for(&page), Err(Error::CorruptRecord)); + + let mut sealed = self::sealed(&result); + sealed.envelope.plaintext_length += 1; + sealed.envelope.ciphertext_length += 1; + assert!(sealed.validate_for(&page).is_err()); + } +} From 8ab58e39febb63e59f66b0cc71bb5a7f839349ef Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 20:25:28 +0000 Subject: [PATCH 35/87] test(racer): cover GPU decryption of sealed gpu-read pages --- .../src/client/tests/direct/gpu.rs | 368 ++++++++++++++++++ cmd/racer-dataplane/src/read/range_stream.rs | 64 +++ cmd/racer-dataplane/src/read/sealed.rs | 7 +- cmd/racer-dataplane/src/test_support.rs | 107 ++++- 4 files changed, 541 insertions(+), 5 deletions(-) diff --git a/cmd/racer-dataplane/src/client/tests/direct/gpu.rs b/cmd/racer-dataplane/src/client/tests/direct/gpu.rs index dbe92b6de..90c529b09 100644 --- a/cmd/racer-dataplane/src/client/tests/direct/gpu.rs +++ b/cmd/racer-dataplane/src/client/tests/direct/gpu.rs @@ -241,3 +241,371 @@ fn gpu_mode_resolves_against_hardware() { ); } } + +/// A helper on one simulated GPU with a staging cap, after `setup` runs on +/// the fresh simulation. +fn start_with(staging: u64, setup: fn(&Simulation)) -> Arc { + GpuHelper::start_with_staging( + GpuMode::Required, + staging, + Box::new(move || { + let simulation = Simulation::new(vec![SimDevice::new(0)]); + let environment = simulation.enter(); + setup(&simulation); + SIM.with(|sim| *sim.borrow_mut() = Some((simulation, environment))); + racer_gpu::Gpu::load() + }), + ) + .unwrap() + .unwrap() +} + +fn page_id(number: u64) -> crate::model::PageId { + crate::model::PageId { + version: ObjectVersion { + object: ObjectId { + cache: definition().id, + key: CacheKey([0; 32]), + }, + etag: StrongEtag::parse(b"\"v1\"").unwrap(), + }, + number: crate::model::PageNumber(number), + } +} + +fn worker(fixture: &Fixture) -> &crate::test_support::ReadWorker { + fixture.worker.as_ref().unwrap() +} + +/// Calls to `operation` in the trace from `from` on. +fn calls(helper: &GpuHelper, from: usize, operation: Operation) -> usize { + helper.run(move || { + sim().trace()[from..] + .iter() + .filter(|event| event.operation == operation) + .count() + }) +} + +fn trace_len(helper: &GpuHelper) -> usize { + helper.run(|| sim().trace().len()) +} + +fn counter(value: &std::sync::atomic::AtomicU64) -> u64 { + value.load(std::sync::atomic::Ordering::Relaxed) +} + +/// Read `range` of the object into a fresh allocation at offset 64. +fn gpu_read( + fixture: &Fixture, + helper: &GpuHelper, + range: Option<(u64, u64)>, + length: u64, +) -> (String, Vec<(u8, u64, u64, u32)>, OwnedFd, u64) { + let size = (length + 64).next_multiple_of(GRANULE); + let allocation = allocate(helper, 0, size); + let header = range.map_or(String::new(), |(start, end)| { + format!("Range: bytes={start}-{end}\r\n") + }); + let output = exchange( + fixture, + &direct_request("gpu-read", &fields(&header, 64, length, size)), + &[allocation.as_raw_fd()], + ); + let (head, body) = split(&output); + let frames = frames(body); + settle(fixture); + (head, frames, allocation, size) +} + +/// Fill the page cache with an ordinary read, then leave each page's +/// ciphertext in memory, on disk, or both, and no plaintext. +fn arrange( + fixture: &Fixture, + helper: &GpuHelper, + pages: u64, + length: u64, + memory: bool, + disk: bool, +) { + let worker = worker(fixture); + worker.retain(true); + if disk { + worker.open_disk(); + } + let (head, _, _, _) = gpu_read(fixture, helper, None, length); + assert!(head.starts_with("http/1.1 200"), "{head}"); + let mut copies = Vec::new(); + for number in 0..pages { + let result = worker.memory().get(&page_id(number)).unwrap().unwrap(); + copies.push(result.copy()); + } + worker.memory().remove_cache(&definition().id).unwrap(); + for copy in copies { + if disk { + worker.persist(copy.clone()); + } + if memory { + worker + .memory() + .publish_ciphertext(crate::memory::UnverifiedPage { + copy, + disk_token: None, + }) + .unwrap(); + } + } +} + +/// Nothing is held: no imports, streams, leaks or staged slots. +fn assert_idle(fixture: &Fixture, helper: &GpuHelper, slots: usize) { + settle(fixture); + assert_released(helper); + assert_eq!(helper.free_slots(0), slots); +} + +fn body(length: usize) -> Vec { + (0..length).map(|i| (i * 7 % 251) as u8).collect() +} + +/// Slots left for staged ciphertext with the default staging cap. +fn default_slots() -> usize { + (crate::config::DEFAULT_GPU_STAGING_BYTES / crate::gpu::SLOT_BYTES) as usize - 2 +} + +#[test] +fn gpu_read_memory_plaintext_hit_copies_plaintext() { + let helper = start(GpuMode::Required, 1).unwrap(); + let data = body(5000); + let fixture = gpu_fixture(data.clone(), &helper); + worker(&fixture).retain(true); + let _ = gpu_read(&fixture, &helper, None, 5000); + let mark = trace_len(&helper); + let decrypts = worker(&fixture).cpu_decrypts(); + let (head, frames, allocation, _) = gpu_read(&fixture, &helper, Some((10, 99)), 90); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert_eq!(frames, [(1, 0, 10, 90), (2, 1, 90, 0)]); + assert_eq!(read_allocation(&allocation, 64, 90), &data[10..100]); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 0); + assert_eq!(calls(&helper, mark, Operation::CopyH2d), 1); + assert_eq!(worker(&fixture).cpu_decrypts(), decrypts); + assert_eq!(counter(&helper.counters().decrypted), 0); + assert_idle(&fixture, &helper, default_slots()); +} + +#[test] +fn gpu_read_retained_ciphertext_decrypts_part_of_a_page_on_the_gpu() { + let helper = start(GpuMode::Required, 1).unwrap(); + let data = body(5000); + let fixture = gpu_fixture(data.clone(), &helper); + arrange(&fixture, &helper, 1, 5000, true, false); + let mark = trace_len(&helper); + let (head, frames, allocation, size) = gpu_read(&fixture, &helper, Some((100, 199)), 100); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert_eq!(frames, [(1, 0, 100, 100), (2, 1, 100, 0)]); + // Only the window holds bytes, though the GPU decrypted the whole page. + let all = read_allocation(&allocation, 0, size as usize); + assert_eq!(&all[64..164], &data[100..200]); + assert!(all[..64].iter().chain(&all[164..]).all(|&b| b == 0)); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 1); + assert_eq!(calls(&helper, mark, Operation::CopyD2d), 1); + assert_eq!(worker(&fixture).cpu_decrypts(), 0); + assert_eq!(counter(&helper.counters().decrypted), 1); + assert_idle(&fixture, &helper, default_slots()); +} + +#[test] +fn gpu_read_disk_pages_go_through_cufile_and_gpu_decrypt() { + let length = 2 * PAGE_BYTES as usize + 4099; + let helper = start(GpuMode::Required, 1).unwrap(); + let data = body(length); + let fixture = gpu_fixture(data.clone(), &helper); + arrange(&fixture, &helper, 3, length as u64, false, true); + let mark = trace_len(&helper); + let (head, frames, allocation, _) = gpu_read(&fixture, &helper, None, length as u64); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert_eq!(frames.len(), 4, "{frames:?}"); + assert_eq!(frames[3], (2, 3, length as u64, 0)); + assert!(read_allocation(&allocation, 64, length) == data); + assert_eq!(calls(&helper, mark, Operation::FileRead), 3); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 3); + // Whole pages decrypt straight into the import. + assert_eq!(calls(&helper, mark, Operation::CopyD2d), 0); + assert_eq!(calls(&helper, mark, Operation::CopyH2d), 0); + assert_eq!(worker(&fixture).cpu_decrypts(), 0); + assert_eq!(counter(&helper.counters().staged), 3); + assert_eq!(counter(&helper.counters().decrypted), 3); + assert_idle(&fixture, &helper, default_slots()); +} + +#[test] +fn gpu_read_tampered_disk_ciphertext_fails_and_drops_the_record() { + use std::os::unix::fs::FileExt; + let helper = start(GpuMode::Required, 1).unwrap(); + let data = body(5000); + let fixture = gpu_fixture(data, &helper); + arrange(&fixture, &helper, 1, 5000, false, true); + let location = worker(&fixture).disk_location(&page_id(0)).unwrap(); + let slab = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(worker(&fixture).slab_path()) + .unwrap(); + let mut record = vec![0; location.extent.length()]; + slab.read_exact_at(&mut record, location.extent.offset()) + .unwrap(); + let decoded = crate::store::parse_bytes(&record, location.extent).unwrap(); + let at = location.extent.offset() + decoded.ciphertext.start as u64 + 10; + let mut byte = [0]; + slab.read_exact_at(&mut byte, at).unwrap(); + slab.write_all_at(&[byte[0] ^ 1], at).unwrap(); + slab.sync_all().unwrap(); + let (head, frames, allocation, size) = gpu_read(&fixture, &helper, None, 5000); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert!(frames.is_empty(), "{frames:?}"); + assert!( + read_allocation(&allocation, 0, size as usize) + .iter() + .all(|&b| b == 0) + ); + assert_eq!(counter(&helper.counters().rejected), 1); + // The owner dropped the record, as after a failed CPU decrypt. + assert!(worker(&fixture).disk_location(&page_id(0)).is_none()); + assert_eq!(worker(&fixture).cpu_decrypts(), 0); + assert_idle(&fixture, &helper, default_slots()); +} + +/// Replace the retained ciphertext of page 0 with tampered bytes, keeping +/// the original CRC (`keep_crc`) or the CRC of the tampered bytes. +fn tamper_retained(fixture: &Fixture, keep_crc: bool) { + use crate::memory::CiphertextBytes; + use crate::memory::CiphertextPage; + let memory = worker(fixture).memory(); + let copy = memory.unverified(&page_id(0)).unwrap().unwrap().copy; + let mut bytes = copy.ciphertext.bytes().to_vec(); + bytes[3] ^= 0x40; + let crc = if keep_crc { + copy.ciphertext.checksum() + } else { + racer_crypto::crc64(&bytes) + }; + let reservation = fixture + .listeners + .admission + .reserve( + Some(&definition().id), + crate::admission::ResourceClass::Ciphertext, + bytes.len(), + ) + .unwrap(); + let tampered = CiphertextPage { + provenance: None, + inner: Arc::new(CiphertextBytes { + checksum: std::sync::OnceLock::from(crc), + envelope: copy.ciphertext.envelope().clone(), + storage: flow_control::ChargedBytes { bytes, reservation }.into(), + }), + }; + memory.remove_cache(&definition().id).unwrap(); + memory + .publish_ciphertext(crate::memory::UnverifiedPage { + copy: crate::memory::CiphertextCopy { + metadata: copy.metadata, + ciphertext: tampered, + }, + disk_token: None, + }) + .unwrap(); +} + +#[test] +fn gpu_read_crc_and_tag_mismatches_fail_without_writing() { + for keep_crc in [true, false] { + let helper = start(GpuMode::Required, 1).unwrap(); + let fixture = gpu_fixture(body(5000), &helper); + arrange(&fixture, &helper, 1, 5000, true, false); + tamper_retained(&fixture, keep_crc); + let mark = trace_len(&helper); + let (head, frames, allocation, size) = gpu_read(&fixture, &helper, Some((0, 9)), 10); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert!(frames.is_empty(), "{frames:?}"); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 1); + // A failed verdict copies nothing out of scratch. + assert_eq!(calls(&helper, mark, Operation::CopyD2d), 0); + assert!( + read_allocation(&allocation, 0, size as usize) + .iter() + .all(|&b| b == 0) + ); + assert_eq!(counter(&helper.counters().rejected), 1); + // The rejected copy left memory. + assert!( + worker(&fixture) + .memory() + .unverified(&page_id(0)) + .unwrap() + .is_none() + ); + assert_idle(&fixture, &helper, default_slots()); + } +} + +#[test] +fn gpu_read_without_free_staging_reads_disk_through_host_memory() { + // Room for the two scratch slots only. + let staging = (2 * crate::gpu::SLOT_BYTES).next_multiple_of(crate::config::GPU_STAGING_UNIT); + let helper = start_with(staging, |_| {}); + let data = body(5000); + let fixture = gpu_fixture(data.clone(), &helper); + arrange(&fixture, &helper, 1, 5000, false, true); + let mark = trace_len(&helper); + let (head, frames, allocation, _) = gpu_read(&fixture, &helper, None, 5000); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert_eq!(frames, [(1, 0, 0, 5000), (2, 1, 5000, 0)]); + assert_eq!(read_allocation(&allocation, 64, 5000), data); + assert_eq!(calls(&helper, mark, Operation::FileRead), 0); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 1); + assert!(counter(&helper.counters().exhausted) >= 1); + assert_eq!(worker(&fixture).cpu_decrypts(), 0); + assert_idle(&fixture, &helper, 0); +} + +#[test] +fn gpu_read_falls_back_to_host_ciphertext_when_cufile_fails() { + let helper = start(GpuMode::Required, 1).unwrap(); + let data = body(5000); + let fixture = gpu_fixture(data.clone(), &helper); + arrange(&fixture, &helper, 1, 5000, false, true); + helper.run(|| sim().fault(Operation::FileRead, -5)); + let mark = trace_len(&helper); + let (head, frames, allocation, _) = gpu_read(&fixture, &helper, None, 5000); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert_eq!(frames, [(1, 0, 0, 5000), (2, 1, 5000, 0)]); + assert_eq!(read_allocation(&allocation, 64, 5000), data); + assert_eq!(calls(&helper, mark, Operation::FileRead), 1); + assert_eq!(calls(&helper, mark, Operation::CopyH2d), 1); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 1); + assert_eq!(counter(&helper.counters().file_failures), 1); + assert_eq!(worker(&fixture).cpu_decrypts(), 0); + assert_idle(&fixture, &helper, default_slots()); +} + +#[test] +fn gpu_read_without_decrypt_kernels_decrypts_on_the_cpu() { + let helper = start_with(crate::config::DEFAULT_GPU_STAGING_BYTES, |simulation| { + simulation.disable_kernels() + }); + assert!(!helper.decrypts(0)); + let data = body(5000); + let fixture = gpu_fixture(data.clone(), &helper); + arrange(&fixture, &helper, 1, 5000, false, true); + let mark = trace_len(&helper); + let (head, frames, allocation, _) = gpu_read(&fixture, &helper, None, 5000); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert_eq!(frames, [(1, 0, 0, 5000), (2, 1, 5000, 0)]); + assert_eq!(read_allocation(&allocation, 64, 5000), data); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 0); + assert_eq!(calls(&helper, mark, Operation::FileRead), 0); + assert_eq!(worker(&fixture).cpu_decrypts(), 1); + assert_idle(&fixture, &helper, default_slots()); +} diff --git a/cmd/racer-dataplane/src/read/range_stream.rs b/cmd/racer-dataplane/src/read/range_stream.rs index bb7f88fea..0e703279a 100644 --- a/cmd/racer-dataplane/src/read/range_stream.rs +++ b/cmd/racer-dataplane/src/read/range_stream.rs @@ -1728,6 +1728,70 @@ pub(super) mod tests { } } + #[test] + fn sealed_pages_are_delivered_in_order_only_to_direct_readers() { + let f = Fixture::new(crate::test_support::cluster::config(false).limits, 2); + let admission = &f.admission; + let mut metadata = metadata(); + metadata.length = 2 * PAGE_BYTES; + let range = ByteRange::Closed { + first: 5, + last: PAGE_BYTES + 9, + } + .resolve(metadata.length) + .unwrap(); + let scope = || { + RequestScope::new(RequestId([9; 16]), Instant::now() + Duration::from_secs(60)).unwrap() + }; + let mut stream = f.open(&metadata, range, scope()); + stream.configure_subscription(4, 4 * PAGE_BYTES).unwrap(); + let first = crate::read::sealed::tests::sealed(&page_result(admission, &metadata, 0)); + stream.inject(Slot::Sealed(Box::new(first))); + stream.inject(Slot::Ready(Ok(page_result(admission, &metadata, 1)))); + match futures::executor::block_on(stream.next_direct()).unwrap() { + Some(DirectItem::Sealed(page, slice)) => { + assert_eq!(page.page().number, PageNumber(0)); + assert_eq!( + (slice.page, slice.offset, slice.length), + (PageNumber(0), 5, PAGE_BYTES as u32 - 5) + ); + } + _ => panic!("expected a sealed page"), + } + stream + .release_page(PageNumber(0), PAGE_BYTES as u32 - 5) + .unwrap(); + match futures::executor::block_on(stream.next_direct()).unwrap() { + Some(DirectItem::Plaintext(reader)) => { + assert_eq!(reader.slice().page, PageNumber(1)); + assert_eq!(reader.slice().length, 10); + } + _ => panic!("expected plaintext"), + } + assert!( + futures::executor::block_on(stream.next_direct()) + .unwrap() + .is_none() + ); + + // A sealed page for another page number fails that page. + let mut stream = f.open(&metadata, range, scope()); + stream.configure_subscription(4, 4 * PAGE_BYTES).unwrap(); + let other = crate::read::sealed::tests::sealed(&page_result(admission, &metadata, 1)); + stream.inject(Slot::Sealed(Box::new(other))); + assert!(futures::executor::block_on(stream.next_direct()).is_err()); + + // A Get stream never yields sealed pages. + let mut stream = f.open(&metadata, range, scope()); + stream.configure_subscription(4, 4 * PAGE_BYTES).unwrap(); + let first = crate::read::sealed::tests::sealed(&page_result(admission, &metadata, 0)); + stream.inject(Slot::Sealed(Box::new(first))); + assert_eq!( + futures::executor::block_on(stream.next_slice()).err(), + Some(Error::Internal) + ); + } + #[test] fn retained_boundary_slices_keep_full_plaintext_charged_until_exact_release() { let f = Fixture::new(crate::test_support::cluster::config(false).limits, 2); diff --git a/cmd/racer-dataplane/src/read/sealed.rs b/cmd/racer-dataplane/src/read/sealed.rs index a00127495..772fa7ef3 100644 --- a/cmd/racer-dataplane/src/read/sealed.rs +++ b/cmd/racer-dataplane/src/read/sealed.rs @@ -143,15 +143,16 @@ impl SealedPage { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use crate::memory::CiphertextCopy; - fn sealed(result: &PageResult) -> SealedPage { + /// A host-sourced sealed page for `result`, with a fixed test key. + pub(crate) fn sealed(result: &PageResult) -> SealedPage { let envelope = result.ciphertext.envelope().clone(); SealedPage { metadata: result.metadata.clone(), - aad: crate::security::page_aad(&envelope).unwrap(), + aad: crate::security::page_aad(&envelope).unwrap_or_default(), key: racer_crypto::xaes_derive(&[7; 32], &envelope.nonce.0), expected_crc: Some(result.ciphertext.checksum()), envelope, diff --git a/cmd/racer-dataplane/src/test_support.rs b/cmd/racer-dataplane/src/test_support.rs index 1c2537d1e..8eb3053e9 100644 --- a/cmd/racer-dataplane/src/test_support.rs +++ b/cmd/racer-dataplane/src/test_support.rs @@ -491,6 +491,12 @@ pub(crate) struct ReadWorker { writer: Rc, memory: Rc, cache: racer_control_wire::CacheId, + admission: Rc>, + clock: Rc, + reactor: Rc, + metrics: crate::telemetry::Metrics, + /// Keep cached pages and queued disk writes between polls. + retain: std::cell::Cell, } /// Publish a topology-only fixture through the real generic publication authority. @@ -555,7 +561,7 @@ impl ReadWorker { let segments = Rc::new(page_alloc::Segments::new(64 * 1024 * 1024)); // Reads use an empty disk index. No slabs need to be opened or written. let slabs = Rc::new(page_alloc::Slab::new( - origin.root.join("slabs/worker-0-slab-0.dat"), + Self::disk_root(&origin).join("slabs/worker-0-slab-0.dat"), 256 * 1024 * 1024, 64 * 1024 * 1024, crate::model::PAGE_BYTES as usize + crate::store::MAX_HEADER_BYTES + 16, @@ -568,8 +574,9 @@ impl ReadWorker { reactor.clone(), availability.clone(), )); + let clock = Rc::new(SegmentClock::new(index.clone(), segments.clone(), 1)); let disk = Rc::new(StoreReader::new( - Rc::new(SegmentClock::new(index.clone(), segments.clone(), 1)), + clock.clone(), index.clone(), segments, slabs, @@ -580,6 +587,10 @@ impl ReadWorker { let (port, engine) = crate::security::pair(WorkerId(0), 0, std::num::NonZeroUsize::new(16).unwrap()); let crypto = Rc::new(CryptoClient::new(port)); + let metrics = crate::telemetry::Metrics::default(); + crypto.set_metrics(metrics.clone()); + let worker_admission = admission.clone(); + let worker_reactor = reactor.clone(); let credentials = Rc::new(CredentialCrypto::new(keys.clone(), admission.clone())); let candidates = Rc::new(CandidatePolicy::new( keys.node().clone(), @@ -642,9 +653,98 @@ impl ReadWorker { writer, memory, cache: cache.id, + admission: worker_admission, + clock, + reactor: worker_reactor, + metrics, + retain: std::cell::Cell::new(false), } } + /// Keep cached pages and queued disk writes between polls, so a test can + /// arrange memory and disk contents across requests. + pub fn retain(&self, retain: bool) { + self.retain.set(retain); + } + + /// This worker's page cache. + pub fn memory(&self) -> &Rc { + &self.memory + } + + /// CPU page decryptions started on this worker's crypto engine. + pub fn cpu_decrypts(&self) -> u64 { + self.metrics + .count(crate::telemetry::Event::CryptoDecryptStarted) + } + + /// Open this worker's slab so pages can be written to and read from disk. + pub fn open_disk(&self) { + std::fs::create_dir_all(Self::disk_root(&self.origin).join("slabs")).unwrap(); + self.writer + .configure(self.admission.clone(), self.clock.clone(), 8, 64) + .unwrap(); + let _ = futures::executor::block_on(self.writer.open()).unwrap(); + } + + /// Write `copy` to disk and wait until it is published. + pub fn persist(&self, copy: crate::memory::CiphertextCopy) { + use crate::admission::ResourceClass; + let id = copy.ciphertext.envelope().page.clone(); + let dirty = self + .admission + .reserve( + Some(&id.version.object.cache), + ResourceClass::DirtyCiphertext, + copy.ciphertext.bytes().len(), + ) + .unwrap(); + self.writer.enqueue(copy, dirty).unwrap(); + let scope = crate::runtime::RequestScope::new( + crate::model::RequestId([1; 16]), + std::time::Instant::now() + std::time::Duration::from_secs(10), + ) + .unwrap(); + let mut progress = self.writer.progress(1, &scope); + let waker = futures::task::noop_waker(); + let mut cx = Context::from_waker(&waker); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while self.writer.index().lookup(&id).unwrap().is_none() { + if progress.as_mut().poll(&mut cx).is_ready() { + progress = self.writer.progress(1, &scope); + } + self.reactor.poll_budgeted(32).unwrap(); + assert!( + std::time::Instant::now() < deadline, + "disk write did not finish" + ); + std::thread::yield_now(); + } + } + + /// The published disk location of a page, if any. + pub fn disk_location( + &self, + page: &crate::model::PageId, + ) -> Option { + self.writer + .index() + .lookup(page) + .unwrap() + .map(|entry| entry.location) + } + + /// The slab file backing this worker's disk. + pub fn slab_path(&self) -> std::path::PathBuf { + Self::disk_root(&self.origin).join("slabs/worker-0-slab-0.dat") + } + + /// The origin's directory by its real path. The slab refuses to open + /// through the `/proc` symlink that `root` is. + fn disk_root(origin: &AdapterOrigin) -> std::path::PathBuf { + std::fs::read_link(&origin.root).unwrap_or_else(|_| origin.root.clone()) + } + /// Whether every command submitted to this worker has completed. pub fn is_drained(&self) -> bool { self.endpoint.borrow().is_drained() && self.drivers.pending() == 0 @@ -661,6 +761,9 @@ impl ReadWorker { ) .unwrap(); self.crypto.poll_budgeted(64).unwrap(); + if self.retain.get() { + return; + } // Exercise acquisition/authentication, not persistence or cache retention. // Delivered leases remain charged even after the cache drops its copy. self.writer.discard_unsubmitted(); From 8d20b24e43d6b0d67943ff5f740dcefef85ab10f Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 20:29:14 +0000 Subject: [PATCH 36/87] chore(racer): satisfy clippy for sealed gpu-read pages --- cmd/racer-dataplane/src/client/tests/direct/gpu.rs | 10 ++++------ cmd/racer-dataplane/src/read/sealed.rs | 2 ++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/racer-dataplane/src/client/tests/direct/gpu.rs b/cmd/racer-dataplane/src/client/tests/direct/gpu.rs index 90c529b09..6e7dd487c 100644 --- a/cmd/racer-dataplane/src/client/tests/direct/gpu.rs +++ b/cmd/racer-dataplane/src/client/tests/direct/gpu.rs @@ -295,13 +295,11 @@ fn counter(value: &std::sync::atomic::AtomicU64) -> u64 { value.load(std::sync::atomic::Ordering::Relaxed) } +/// A response head, its frames, the allocation and its size. +type Read = (String, Vec<(u8, u64, u64, u32)>, OwnedFd, u64); + /// Read `range` of the object into a fresh allocation at offset 64. -fn gpu_read( - fixture: &Fixture, - helper: &GpuHelper, - range: Option<(u64, u64)>, - length: u64, -) -> (String, Vec<(u8, u64, u64, u32)>, OwnedFd, u64) { +fn gpu_read(fixture: &Fixture, helper: &GpuHelper, range: Option<(u64, u64)>, length: u64) -> Read { let size = (length + 64).next_multiple_of(GRANULE); let allocation = allocate(helper, 0, size); let header = range.map_or(String::new(), |(start, end)| { diff --git a/cmd/racer-dataplane/src/read/sealed.rs b/cmd/racer-dataplane/src/read/sealed.rs index 772fa7ef3..a5818255c 100644 --- a/cmd/racer-dataplane/src/read/sealed.rs +++ b/cmd/racer-dataplane/src/read/sealed.rs @@ -40,6 +40,7 @@ impl SealedTarget { } /// One fetched window page in sealed mode. +#[allow(clippy::large_enum_variant)] // Pages move unboxed through the mailbox, as `Value::Page` does. pub enum Fetched { /// Verified plaintext. Page(PageResult), @@ -49,6 +50,7 @@ pub enum Fetched { } /// Where a sealed page's ciphertext is. +#[allow(clippy::large_enum_variant)] // Always inside a boxed SealedPage. pub enum SealedSource { /// Ciphertext and tag in host memory: retained, pending, read from disk /// through the slab, or received from a peer. From 8fc1743432fa92306fb2261bea5e30e6552fde2a Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Thu, 22 Oct 2026 20:29:50 +0000 Subject: [PATCH 37/87] docs(racer): describe GPU decryption of gpu-read pages --- designs/racer-gpudirect.md | 39 +++++++++++++++++++++++++++++++++ docs/content/reference/racer.md | 23 ++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/designs/racer-gpudirect.md b/designs/racer-gpudirect.md index 3459bed27..f43e521dc 100644 --- a/designs/racer-gpudirect.md +++ b/designs/racer-gpudirect.md @@ -196,6 +196,45 @@ Ciphertext reaches GPU staging memory owned by the dataplane in one of three way `RACER_GPU_STAGING_BYTES` caps the dataplane's ciphertext staging memory per GPU. +How the implementation works today, where it adds to or differs from the plan +above: + +- The read path asks for sealed pages only for `gpu-read` + (`RangeStream::set_sealed`, `Fill::acquire_sealed`). Pages that are verified + plaintext in memory, origin fills (the CPU encrypts them, so it has the + plaintext), and pages another reader already decrypted are still delivered + as plaintext and copied host to GPU. So are pages that a peer selection fans + out to the read window, and pages whose key cannot be derived or whose AAD is + longer than the kernel accepts (4 KiB). +- The XAES key derivation runs on the page's owning I/O thread + (`PageCrypto::derive_page_key`), not on the crypto worker. It is three AES + blocks. +- Disk records read through host memory keep the CPU CRC check of the + existing fill path; the GPU checks the CRC again. Records read with cuFile + are checked only on the GPU, against the CRC in the record header. +- Each GPU has one staging buffer of 16 MiB + 4 KiB slots, allocated on first + use. Two slots are scratch for the helper's current job: host ciphertext, + and the plaintext of a page the range covers only in part. The GPU decrypts + the whole page there (the tag covers all of it), the helper checks the + verdict, then copies only the requested bytes into the client's window. A + whole page decrypts straight into the window. The scratch plaintext is not + wiped; it is dataplane-owned GPU memory that is never read back to the host. +- The other slots hold cuFile reads. The page's owning worker reads the record + header through host memory (`StoreReader::direct_read`), holds the segment + lease, and asks the GPU helper to read the ciphertext into a free slot. When + no slot is free, cuFile fails, or the record is v1, unaligned, or on a + simulated descriptor, the worker reads the page through host memory instead. + A cuFile driver or registration failure turns cuFile off until restart. +- At startup the helper runs a zero-length decrypt on each GPU. A GPU whose + shim has no decrypt kernels, or a staging cap smaller than the two scratch + slots, gets plaintext pages only. +- When the GPU rejects a page (CRC or tag), the read fails without a page frame + and the page's owner drops that copy from memory and disk, as it does after a + failed CPU decrypt. When it accepts a page that came from a peer, the owner + may persist it, as it does after a CPU decrypt. +- RDMA peers do not yet write into GPU staging: an RDMA page lands in host + memory and takes one host-to-GPU copy. + ### Disk layout v2 The store already writes format version 2 records (see diff --git a/docs/content/reference/racer.md b/docs/content/reference/racer.md index 9b74eb661..f8c1e9612 100644 --- a/docs/content/reference/racer.md +++ b/docs/content/reference/racer.md @@ -203,7 +203,7 @@ Numeric values are unsigned decimal strings: `268435456`, not `256Mi`. | `RACER_ALLOW_SMT` | `false` | Use eligible physical cores by default; `true` opts into logical CPUs | | `RACER_ENABLE_RDMA` | `auto` | `auto`, `true`, or `false`; see [modes](#rdma) | | `RACER_ENABLE_GPU` | `auto` | `auto`, `true`, or `false`; see [GPU reads](#gpu-reads) | -| `RACER_GPU_STAGING_BYTES` | `268435456` (256 MiB) | Cap on ciphertext staging memory per GPU, in bytes. Must be a nonzero multiple of 2 MiB. Not used yet | +| `RACER_GPU_STAGING_BYTES` | `268435456` (256 MiB) | Cap on ciphertext staging memory per GPU, in bytes. Must be a nonzero multiple of 2 MiB. Below about 34 MiB the GPU does not decrypt and the CPU does; see [GPU reads](#gpu-reads) | | `RACER_SHARES` | `4` | Enrollment proposal; explicit Node `shares` wins | | `RACER_PLAINTEXT_BYTES` / `RACER_CIPHERTEXT_BYTES` | Unset: `67108864` (64 MiB) / `134217728` (128 MiB) per final I/O worker | Independent plaintext/ciphertext admission budgets; an explicit value is node-wide and divided among I/O workers | | `RACER_DIRTY_BYTES` | Unset: `167772160` (160 MiB) per final I/O worker | Dirty-data budget; an explicit value is node-wide | @@ -1025,6 +1025,27 @@ NUMA node, compute mode, and whether it can use it. It skips GPUs in `EXCLUSIVE_PROCESS` or `PROHIBITED` mode. One helper thread does all GPU work for the process. At most 256 GPU reads run at once; more get 503. +Where the bytes come from decides who decrypts them: + +- A page that is already plaintext in memory is copied to the GPU. +- A page fetched from the origin is encrypted on the CPU, so its plaintext is + copied to the GPU too. +- Other pages reach the GPU still encrypted, and the GPU checks and decrypts + them. This covers pages kept encrypted in memory, pages on local disk, and + pages from peers. The CPU does not decrypt them. +- A page on local disk is read straight from the disk into GPU memory with + cuFile when it can be. Otherwise the CPU reads it and copies it to the GPU. + +If the GPU finds a damaged page, the read fails and the dataplane drops that +copy from memory or disk, as it does when the CPU finds one. + +`RACER_GPU_STAGING_BYTES` is the GPU memory the dataplane keeps on each GPU for +encrypted pages. It is split into slots of one page (16 MiB plus 4 KiB). Two +slots are needed for decryption, so a smaller cap means the CPU decrypts. The +other slots hold pages read with cuFile. When all of them are in use, the +dataplane reads the next page through host memory instead. If the image has no +GPU decrypt support, the CPU decrypts every page. + The dataplane does not request `nvidia.com/gpu`. It is privileged, so it already sees `/dev/nvidia*`. It only needs the host driver libraries. `RACER_DATAPLANE_NVIDIA` in `racer-config` controls how it asks for them: From aab1fff968ff049d59aa36acfe4c1c8848cd767f Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:23:22 +0000 Subject: [PATCH 38/87] test(racer): close pyproject.toml file handle in entry point test --- clients/python/racer-vllm/tests/test_loader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clients/python/racer-vllm/tests/test_loader.py b/clients/python/racer-vllm/tests/test_loader.py index c92ff31b4..eec4c1c07 100644 --- a/clients/python/racer-vllm/tests/test_loader.py +++ b/clients/python/racer-vllm/tests/test_loader.py @@ -183,6 +183,7 @@ def named_parameters(self): def test_entry_point_declared(): here = os.path.dirname(__file__) - text = open(os.path.join(here, "..", "pyproject.toml")).read() + with open(os.path.join(here, "..", "pyproject.toml"), encoding="utf-8") as f: + text = f.read() assert '[project.entry-points."vllm.general_plugins"]' in text assert 'racer = "racer_vllm:register"' in text From d7115b10e746fee2af2874a820b31119038df310 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:23:32 +0000 Subject: [PATCH 39/87] test(racer): assert vLLM loader register() registers only once --- clients/python/racer-vllm/tests/test_loader.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/clients/python/racer-vllm/tests/test_loader.py b/clients/python/racer-vllm/tests/test_loader.py index eec4c1c07..710663cee 100644 --- a/clients/python/racer-vllm/tests/test_loader.py +++ b/clients/python/racer-vllm/tests/test_loader.py @@ -141,6 +141,23 @@ def wrap(cls): return registry +def test_register_is_idempotent(monkeypatch): + install_fake_vllm(monkeypatch) + ml = sys.modules["vllm.model_executor.model_loader"] + inner = ml.register_model_loader + calls = [] + + def counting(name): + calls.append(name) + return inner(name) + + ml.register_model_loader = counting + loader.register() + loader.register() + assert calls == ["racer"] + assert loader._registered is True + + def test_register_and_load(server, uri, monkeypatch, tmp_path): registry = install_fake_vllm(monkeypatch) loader.register() From 54e97e1a95707128cfdbb193a7e3c5d43058fa1d Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:23:38 +0000 Subject: [PATCH 40/87] test(racer): use a single import style for the racer module --- clients/python/racer-vllm/src/racer_vllm/weights.py | 3 ++- clients/python/racer/tests/conftest.py | 4 ++-- clients/python/racer/tests/fakeserver.py | 2 +- clients/python/racer/tests/test_client.py | 6 +++--- clients/python/racer/tests/test_wire_golden.py | 6 +++--- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/clients/python/racer-vllm/src/racer_vllm/weights.py b/clients/python/racer-vllm/src/racer_vllm/weights.py index f4bc4e2fe..1c14fca8b 100644 --- a/clients/python/racer-vllm/src/racer_vllm/weights.py +++ b/clients/python/racer-vllm/src/racer_vllm/weights.py @@ -172,7 +172,8 @@ def __init__(self, device: int | None = None, clone: bool = True, use_vmm: bool self._events: dict = {} def _new(self, size: int): - from racer import cuda, tensors + import racer.cuda as cuda + import racer.tensors as tensors if self.use_vmm: try: diff --git a/clients/python/racer/tests/conftest.py b/clients/python/racer/tests/conftest.py index 30c59c840..4478c36e0 100644 --- a/clients/python/racer/tests/conftest.py +++ b/clients/python/racer/tests/conftest.py @@ -10,14 +10,14 @@ from fakeserver import FakeRacer # noqa: E402 import racer # noqa: E402 -from racer import _wire # noqa: E402 +import racer._wire # noqa: E402 SMALL_PAGE = 4096 @pytest.fixture def small_pages(monkeypatch): - monkeypatch.setattr(_wire, "PAGE_SIZE", SMALL_PAGE) + monkeypatch.setattr(racer._wire, "PAGE_SIZE", SMALL_PAGE) return SMALL_PAGE diff --git a/clients/python/racer/tests/fakeserver.py b/clients/python/racer/tests/fakeserver.py index bbe3d0a38..aa2f017fb 100644 --- a/clients/python/racer/tests/fakeserver.py +++ b/clients/python/racer/tests/fakeserver.py @@ -13,7 +13,7 @@ from dataclasses import dataclass, field import racer -from racer import _wire as wire +import racer._wire as wire @dataclass diff --git a/clients/python/racer/tests/test_client.py b/clients/python/racer/tests/test_client.py index 48fccd303..071daee8f 100644 --- a/clients/python/racer/tests/test_client.py +++ b/clients/python/racer/tests/test_client.py @@ -7,8 +7,8 @@ from fakeserver import Obj import racer -from racer import _wire as wire -from racer.client import _mode_state +import racer._wire as wire +import racer.client DATA = bytes(range(256)) * 50 # 12800 bytes: 4 pages of 4096 (last partial) @@ -253,7 +253,7 @@ def test_modes_remembered_per_socket(server, tmp_path): server.old = True with racer.HostBuffer(len(DATA)) as buf: client(server).read_into_host(req, buf) - assert _mode_state[server.path][racer.HOST] == "unsupported" + assert racer.client._mode_state[server.path][racer.HOST] == "unsupported" other = racer.Client(socket_path=str(tmp_path / "other")) assert other.mode_state(racer.HOST) is None diff --git a/clients/python/racer/tests/test_wire_golden.py b/clients/python/racer/tests/test_wire_golden.py index 4dd2e103f..a8d5a2246 100644 --- a/clients/python/racer/tests/test_wire_golden.py +++ b/clients/python/racer/tests/test_wire_golden.py @@ -14,7 +14,7 @@ import pytest import racer -from racer import _wire as wire +import racer._wire as wire KEY = "ab" * 32 @@ -108,9 +108,9 @@ def test_direct_head_from_rust_is_accepted(): "Racer-Delivery: host\r\nConnection: close\r\n\r\n" % length ).encode() h = wire.parse_response_head(raw) - from racer.client import Client + import racer.client - meta, first, end = Client._check_selection(h, 3, 3, None) + meta, first, end = racer.client.Client._check_selection(h, 3, 3, None) assert (meta.size, first, end) == (length, 3, 6) assert wire.decimal(h.get("content-length")) == wire.FRAME_SIZE * (wire.page_count(first, end) + 1) seq = wire.Sequence(first, end) From 1b8b30e4f70e4c29607b8298dcbc79f32f80d5df Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:23:46 +0000 Subject: [PATCH 41/87] test(racer): use consistent returns in fake server dispatch --- clients/python/racer/tests/fakeserver.py | 51 ++++++++++++++++-------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/clients/python/racer/tests/fakeserver.py b/clients/python/racer/tests/fakeserver.py index aa2f017fb..386117a83 100644 --- a/clients/python/racer/tests/fakeserver.py +++ b/clients/python/racer/tests/fakeserver.py @@ -137,54 +137,69 @@ def _handle(self, conn) -> None: def _dispatch(self, conn, method, target, headers, fds, seen, rest): if headers.get("host") != "racer" or not target.startswith(wire.OBJECT_PREFIX): - return self._error(conn, 400) + self._error(conn, 400) + return tail = target[len(wire.OBJECT_PREFIX) :] key, _, mode = tail.partition("/") if mode and (self.old or mode not in ("host-read", "gpu-read")): - return self._error(conn, 400) + self._error(conn, 400) + return # Order follows the dataplane: RequestParser::parse # (object-wire/src/client.rs), then direct::open_sink and # HostSink::open (src/client/direct.rs), then the object read. if mode and method != "POST": - return self._error(conn, 405) + self._error(conn, 405) + return if mode: if headers.get("content-length") != "0": - return self._error(conn, 400) + self._error(conn, 400) + return try: off = wire.decimal(headers.get("racer-destination-offset")) length = wire.decimal(headers.get("racer-destination-length")) alloc = wire.decimal(headers.get("racer-allocation-size")) if mode == "gpu-read" else None except racer.RacerError: - return self._error(conn, 400) + self._error(conn, 400) + return if length == 0 or off + length > wire.MAX_WIRE_INTEGER or (alloc is not None and off + length > alloc): - return self._error(conn, 400) + self._error(conn, 400) + return if fds and not mode: - return self._error(conn, 400) + self._error(conn, 400) + return if mode and len(fds) != 1: - return self._error(conn, 400) + self._error(conn, 400) + return if mode == "gpu-read" and self.gpu_status != 200: - return self._error(conn, self.gpu_status) + self._error(conn, self.gpu_status) + return if mode == "host-read" and self.host_status: - return self._error(conn, self.host_status) + self._error(conn, self.host_status) + return if mode == "host-read": flags = fcntl.fcntl(fds[0], fcntl.F_GETFL) if seen.fd_kind != "regular" or flags & os.O_ACCMODE == os.O_RDONLY or flags & os.O_APPEND: - return self._error(conn, 400) + self._error(conn, 400) + return obj = self.objects.get(key) if obj is None: - return self._error(conn, 404) + self._error(conn, 404) + return if "if-match" in headers and headers["if-match"] != obj.etag: - return self._error(conn, 412) + self._error(conn, 412) + return meta = [("ETag", obj.etag), ("Racer-Expires-At", str(obj.expires_ms))] if obj.content_type: meta.append(("Racer-Content-Type", obj.content_type)) size = len(obj.data) if method == "HEAD": - return self._send( + self._send( conn, 200, meta + [("Content-Length", str(size)), ("Content-Type", "application/octet-stream")] ) + return if method != "POST": - return self._error(conn, 405) + self._error(conn, 405) + return first, end = 0, size if "range" in headers: spec = headers["range"][len("bytes=") :] @@ -192,7 +207,8 @@ def _dispatch(self, conn, method, target, headers, fds, seen, rest): first = int(a) end = min(size, int(b) + 1) if b else size if first >= size and size > 0 or first > size: - return self._error(conn, 416, size) + self._error(conn, 416, size) + return pages = wire.page_count(first, end) sel = meta + [ ("Content-Type", "application/octet-stream"), @@ -203,7 +219,8 @@ def _dispatch(self, conn, method, target, headers, fds, seen, rest): ] if mode: if length < end - first: - return self._error(conn, 400) + self._error(conn, 400) + return self._send( conn, 200, From 4bc4c32b51566216f92547b1262af424746fdc70 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:23:42 +0000 Subject: [PATCH 42/87] refactor(racer): close vLLM loader clients with context managers --- clients/python/racer-vllm/src/racer_vllm/loader.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/clients/python/racer-vllm/src/racer_vllm/loader.py b/clients/python/racer-vllm/src/racer_vllm/loader.py index 2a4b08794..6a21ade40 100644 --- a/clients/python/racer-vllm/src/racer_vllm/loader.py +++ b/clients/python/racer-vllm/src/racer_vllm/loader.py @@ -61,20 +61,16 @@ def _client(self, model_config): def download_model(self, model_config) -> None: # Nothing to download; check the weights are reachable. uri, client = self._client(model_config) - try: + with client: weight_files(client, uri) - finally: - client.close() def load_weights(self, model, model_config) -> None: uri, client = self._client(model_config) s = self.settings - try: + with client: backend = TorchBackend(clone=s["clone"], use_vmm=s["use_vmm"]) weights = iter_weights(client, uri, backend, s["staging_bytes"], s["concurrency"]) loaded = model.load_weights(weights) - finally: - client.close() if loaded is not None and getattr(model_config, "quantization", None) is None: missing = {n for n, _ in model.named_parameters()} - set(loaded) if missing: From 903a1e37f664e4e4e7611536ad2126e1f551ca36 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:25:10 +0000 Subject: [PATCH 43/87] chore(racer): add cc to NOTICE --- NOTICE | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/NOTICE b/NOTICE index d6464eafd..45e368030 100644 --- a/NOTICE +++ b/NOTICE @@ -48,6 +48,15 @@ notices: link: https://docs.rs/crate/base64/0.22.1/source/LICENSE-APACHE - name: MIT License link: https://docs.rs/crate/base64/0.22.1/source/LICENSE-MIT + - dependency: cc + ecosystem: cargo + copyright: + - Copyright (c) 2014 Alex Crichton + license: + - name: Apache License, Version 2.0 + link: https://docs.rs/crate/cc/1.5.1/source/LICENSE-APACHE + - name: MIT License + link: https://docs.rs/crate/cc/1.5.1/source/LICENSE-MIT - dependency: chacha20poly1305 ecosystem: cargo copyright: From ca32343d424bc9f68cde99389b406d534bfad410 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:26:02 +0000 Subject: [PATCH 44/87] fix(racer): explain ignored errors in Python CUDA finalizers --- clients/python/racer/src/racer/cuda.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/clients/python/racer/src/racer/cuda.py b/clients/python/racer/src/racer/cuda.py index 642e56963..24e37bf00 100644 --- a/clients/python/racer/src/racer/cuda.py +++ b/clients/python/racer/src/racer/cuda.py @@ -344,7 +344,10 @@ def __del__(self) -> None: try: self.close() except Exception: - pass + # Finalizers must never raise, and during interpreter shutdown the + # CUDA library or module globals may already be gone. Callers that + # need to observe free errors should call close() explicitly. + return class HostRegistration: From 950427607d9b8e6b2e4149d4a6b749c54df59504 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:26:10 +0000 Subject: [PATCH 45/87] test(racer): explain ignored wakeup errors in the fake server --- clients/python/racer/tests/fakeserver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clients/python/racer/tests/fakeserver.py b/clients/python/racer/tests/fakeserver.py index 386117a83..90e631f29 100644 --- a/clients/python/racer/tests/fakeserver.py +++ b/clients/python/racer/tests/fakeserver.py @@ -67,6 +67,8 @@ def close(self) -> None: c.connect(self.path) c.close() except OSError: + # The connect only wakes the accept loop; if it fails, the + # server is already gone or stopping, so there is nothing to do. pass self._thread.join(5) self.sock.close() From 18594b5241786e78c98440ce29013515733a9099 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:36:29 +0000 Subject: [PATCH 46/87] fix(racer): record vLLM staging events on the target device stream --- .../racer-vllm/src/racer_vllm/weights.py | 8 +++-- .../python/racer-vllm/tests/test_loader.py | 36 +++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/clients/python/racer-vllm/src/racer_vllm/weights.py b/clients/python/racer-vllm/src/racer_vllm/weights.py index 1c14fca8b..5070840e9 100644 --- a/clients/python/racer-vllm/src/racer_vllm/weights.py +++ b/clients/python/racer-vllm/src/racer_vllm/weights.py @@ -211,8 +211,12 @@ def tensor(self, target, rel, info, staged): return raw.view(dtype).view(info.shape) def done(self, target): - ev = self.torch.cuda.Event() - ev.record() + # Record on the target device's stream; the calling thread's current + # device may differ, and its stream would not cover our copies. + cuda = self.torch.cuda + with cuda.device(self.device): + ev = cuda.Event() + ev.record(cuda.current_stream(self.device)) self._events[id(target)] = ev def wait(self, target): diff --git a/clients/python/racer-vllm/tests/test_loader.py b/clients/python/racer-vllm/tests/test_loader.py index 710663cee..d8d765435 100644 --- a/clients/python/racer-vllm/tests/test_loader.py +++ b/clients/python/racer-vllm/tests/test_loader.py @@ -204,3 +204,39 @@ def test_entry_point_declared(): text = f.read() assert '[project.entry-points."vllm.general_plugins"]' in text assert 'racer = "racer_vllm:register"' in text + + +def test_torch_backend_records_event_on_target_device_stream(monkeypatch): + calls = [] + + class Event: + def record(self, stream=None): + calls.append(("record", stream)) + + def synchronize(self): + calls.append(("sync",)) + + class DeviceCtx: + def __init__(self, dev): + self.dev = dev + + def __enter__(self): + calls.append(("enter", self.dev)) + + def __exit__(self, *a): + calls.append(("exit", self.dev)) + + cuda = types.SimpleNamespace( + current_device=lambda: 0, + device=DeviceCtx, + Event=lambda: (calls.append(("event",)), Event())[1], + current_stream=lambda dev=None: ("stream", dev), + ) + monkeypatch.setitem(sys.modules, "torch", types.SimpleNamespace(cuda=cuda)) + from racer_vllm.weights import TorchBackend + + b = TorchBackend(device=3) + target = object() + b.done(target) + b.wait(target) + assert calls == [("enter", 3), ("event",), ("record", ("stream", 3)), ("exit", 3), ("sync",)] From 59f305582ad7ca62950cb7d0660b1549e132d965 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:36:39 +0000 Subject: [PATCH 47/87] fix(racer): fence GPU work before releasing vLLM staging targets --- .../racer-vllm/src/racer_vllm/weights.py | 23 +++++- .../python/racer-vllm/tests/test_loader.py | 71 +++++++++++++++++++ 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/clients/python/racer-vllm/src/racer_vllm/weights.py b/clients/python/racer-vllm/src/racer_vllm/weights.py index 5070840e9..5c56b62fd 100644 --- a/clients/python/racer-vllm/src/racer_vllm/weights.py +++ b/clients/python/racer-vllm/src/racer_vllm/weights.py @@ -66,7 +66,10 @@ def iter_weights( Up to ``depth`` chunks are read ahead, each into its own staging buffer, so reads overlap with the consumer. A staging buffer is reused only after the consumer has moved past all of its tensors and backend.wait() says - queued GPU work that reads it has finished. + queued GPU work that reads it has finished. Every staging and dedicated + target is fenced with done() and wait() before the pipeline drops its + reference, including on early close or error, so external allocations + are never released while queued GPU work may still read them. """ if depth < 1: raise ValueError("depth must be at least 1") @@ -94,19 +97,33 @@ def fill() -> bool: inflight.append((pool.submit(read, req, etag, chunk, target), chunk)) return True + # Dedicated targets the consumer has moved past; fenced at the end. + retired: list = [] + active = None try: while fill(): fut, chunk = inflight.popleft() target = fut.result() + active = target for t in chunk.tensors: yield t.name, backend.tensor(target, t.start - chunk.start, t, not chunk.dedicated) - if not chunk.dedicated: - backend.done(target) + active = None + backend.done(target) + if chunk.dedicated: + retired.append(target) + else: free.append(target) finally: for fut, _ in inflight: fut.cancel() pool.shutdown(wait=True) + if active is not None: + backend.done(active) + retired.append(active) + # Reads have stopped; fence consumer GPU work before release. Targets + # still in flight were fenced by read() and the consumer never saw them. + for target in retired + free: + backend.wait(target) class MemoryBackend(Backend): diff --git a/clients/python/racer-vllm/tests/test_loader.py b/clients/python/racer-vllm/tests/test_loader.py index d8d765435..e3508467a 100644 --- a/clients/python/racer-vllm/tests/test_loader.py +++ b/clients/python/racer-vllm/tests/test_loader.py @@ -240,3 +240,74 @@ def __exit__(self, *a): b.done(target) b.wait(target) assert calls == [("enter", 3), ("event",), ("record", ("stream", 3)), ("exit", 3), ("sync",)] + + +class _FencingBackend(MemoryBackend): + """Records done/wait so tests can check every target is fenced.""" + + def __init__(self): + self.targets = [] + self.done_ids = set() + self.waited = set() + + def _track(self, t): + self.targets.append(t) + return t + + def new_staging(self, size): + return self._track(self.Target(size)) + + def new_dedicated(self, size): + return self._track(self.Target(size)) + + def done(self, target): + self.done_ids.add(id(target)) + self.waited.discard(id(target)) + + def wait(self, target): + if id(target) in self.done_ids: + self.waited.add(id(target)) + + def used(self): + return [t for t in self.targets if id(t) in self.done_ids] + + def assert_fenced(self): + assert self.used() + for t in self.used(): + assert id(t) in self.waited + + +@pytest.mark.parametrize("depth", [1, 2]) +def test_iter_weights_fences_on_completion(server, uri, depth): + setup_model(server, uri) + c = racer.Client(socket_path=server.path, staging_bytes=4096) + b = _FencingBackend() + names = [n for n, _ in iter_weights(c, uri, b, staging_bytes=400, depth=depth)] + assert "w3" in names + b.assert_fenced() + # The dedicated target (w3) was fenced too. + assert any(t.size == 900 for t in b.used()) + + +@pytest.mark.parametrize("stop", ["w1", "w3"]) +def test_iter_weights_fences_on_early_close(server, uri, stop): + setup_model(server, uri) + c = racer.Client(socket_path=server.path, staging_bytes=4096) + b = _FencingBackend() + gen = iter_weights(c, uri, b, staging_bytes=400, depth=2) + for name, _ in gen: + if name == stop: + break + gen.close() + b.assert_fenced() + + +def test_iter_weights_fences_on_exception(server, uri): + setup_model(server, uri) + c = racer.Client(socket_path=server.path, staging_bytes=4096) + b = _FencingBackend() + gen = iter_weights(c, uri, b, staging_bytes=400, depth=2) + next(gen) + with pytest.raises(RuntimeError): + gen.throw(RuntimeError("consumer failed")) + b.assert_fenced() From 81c503113dcc3e0a6d48e830ee21293bc3c83117 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:37:27 +0000 Subject: [PATCH 48/87] fix(racer): bind the allocation's CUDA context for host copies --- clients/python/racer/src/racer/client.py | 17 ++- clients/python/racer/src/racer/cuda.py | 92 +++++++++++----- clients/python/racer/src/racer/host.py | 6 +- clients/python/racer/tests/test_cuda.py | 130 +++++++++++++++++++++++ 4 files changed, 214 insertions(+), 31 deletions(-) diff --git a/clients/python/racer/src/racer/client.py b/clients/python/racer/src/racer/client.py index 0be742e4c..c2146395b 100644 --- a/clients/python/racer/src/racer/client.py +++ b/clients/python/racer/src/racer/client.py @@ -41,6 +41,15 @@ _mode_state: dict[str, dict[str, str]] = {} +def _dest_device(dest) -> int | None: + """Return the CUDA device ordinal of a read destination, if it has one.""" + device = getattr(dest, "device", None) + if isinstance(device, int) and not isinstance(device, bool): + return device + index = getattr(device, "index", None) + return index if isinstance(index, int) else None + + def socket_path_for(cache: str | None = None, socket_path: str | None = None) -> str: """Resolve the client socket path. @@ -199,7 +208,7 @@ def _set_mode(self, mode: str, state: str) -> None: # Staging pool ---------------------------------------------------------- - def _take_staging(self): + def _take_staging(self, device: int | None = None): from .host import HostBuffer with self._pool_lock: @@ -207,7 +216,7 @@ def _take_staging(self): return self._pool.pop() buf = HostBuffer(self.staging_bytes, "racer-staging") if self.pin_staging: - buf.pin() + buf.pin(0 if device is None else device) return buf def _give_staging(self, buf) -> None: @@ -627,7 +636,7 @@ def gpu(ok): return self._run_modes(attempts) def _gpu_via_host(self, request, dest, dest_offset, window, offset, length, if_match, on_page, ok) -> ReadResult: - staging = self._take_staging() + staging = self._take_staging(_dest_device(dest)) try: chunk = staging.size if length: @@ -668,7 +677,7 @@ def frame(o: int, size: int, base=base) -> None: self._give_staging(staging) def _gpu_via_stream(self, request, dest, dest_offset, window, offset, length, if_match, on_page, ok) -> ReadResult: - staging = self._take_staging() + staging = self._take_staging(_dest_device(dest)) try: def write(o: int, piece: memoryview) -> None: diff --git a/clients/python/racer/src/racer/cuda.py b/clients/python/racer/src/racer/cuda.py index 24e37bf00..e56bad471 100644 --- a/clients/python/racer/src/racer/cuda.py +++ b/clients/python/racer/src/racer/cuda.py @@ -37,6 +37,7 @@ from __future__ import annotations +import contextlib import ctypes import logging import os @@ -186,6 +187,16 @@ def ensure_context(device: int = 0) -> None: _check("cuCtxGetCurrent", lib.cuCtxGetCurrent(ctypes.byref(cur))) if cur.value: return + _check("cuCtxSetCurrent", lib.cuCtxSetCurrent(_primary_context(device))) + + +def _primary_context(device: int) -> ctypes.c_void_p: + """Return the device's primary context, retained once per process. + + The retain is never released: torch shares the same primary context and + the cached handle stays valid for the life of the process. + """ + lib = _load() with _lock: ctx = _contexts.get(device) if ctx is None: @@ -194,7 +205,34 @@ def ensure_context(device: int = 0) -> None: ctx = ctypes.c_void_p() _check("cuDevicePrimaryCtxRetain", lib.cuDevicePrimaryCtxRetain(ctypes.byref(ctx), dev.value)) _contexts[device] = ctx + return ctx + + +@contextlib.contextmanager +def bound_context(device: int): + """Make ``device``'s primary context current, restoring the previous one after. + + Worker threads do not inherit a context from the thread that created an + allocation, and an unrelated current context (for example another + device's) is wrong for driver calls on this device's memory, so callers + bind explicitly instead of relying on thread history. Uses + cuCtxGetCurrent/cuCtxSetCurrent + (https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html). + """ + lib = _load() + ctx = _primary_context(device) + prev = ctypes.c_void_p() + _check("cuCtxGetCurrent", lib.cuCtxGetCurrent(ctypes.byref(prev))) + if prev.value == ctx.value: + yield + return _check("cuCtxSetCurrent", lib.cuCtxSetCurrent(ctx)) + try: + yield + finally: + rc = lib.cuCtxSetCurrent(prev) + if rc != 0: + log.warning("racer: restoring the CUDA context failed with CUresult %d", rc) def _address(buf) -> tuple[int, object]: @@ -243,7 +281,6 @@ def __init__(self, size: int, device: int = 0) -> None: if size <= 0: raise RacerError("size must be positive") lib = _load() - ensure_context(device) g = granularity(device) self.device = device self.requested = size @@ -254,23 +291,24 @@ def __init__(self, size: int, device: int = 0) -> None: self._mapped = False prop = _allocation_prop(device) handle = CUmemGenericAllocationHandle() - try: - _check("cuMemCreate", lib.cuMemCreate(ctypes.byref(handle), self.size, ctypes.byref(prop), 0)) - self._handle = handle.value - ptr = CUdeviceptr() - _check("cuMemAddressReserve", lib.cuMemAddressReserve(ctypes.byref(ptr), self.size, g, 0, 0)) - self.ptr = ptr.value - self._reserved = True - _check("cuMemMap", lib.cuMemMap(self.ptr, self.size, 0, self._handle, 0)) - self._mapped = True - desc = CUmemAccessDesc() - desc.location.type = CU_MEM_LOCATION_TYPE_DEVICE - desc.location.id = device - desc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE - _check("cuMemSetAccess", lib.cuMemSetAccess(self.ptr, self.size, ctypes.byref(desc), 1)) - except BaseException: - self.close() - raise + with bound_context(device): + try: + _check("cuMemCreate", lib.cuMemCreate(ctypes.byref(handle), self.size, ctypes.byref(prop), 0)) + self._handle = handle.value + ptr = CUdeviceptr() + _check("cuMemAddressReserve", lib.cuMemAddressReserve(ctypes.byref(ptr), self.size, g, 0, 0)) + self.ptr = ptr.value + self._reserved = True + _check("cuMemMap", lib.cuMemMap(self.ptr, self.size, 0, self._handle, 0)) + self._mapped = True + desc = CUmemAccessDesc() + desc.location.type = CU_MEM_LOCATION_TYPE_DEVICE + desc.location.id = device + desc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE + _check("cuMemSetAccess", lib.cuMemSetAccess(self.ptr, self.size, ctypes.byref(desc), 1)) + except BaseException: + self.close() + raise def export_fd(self) -> int: """Return a new POSIX file descriptor for the allocation. The caller closes it.""" @@ -290,7 +328,8 @@ def copy_from_host(self, offset: int, data) -> None: return addr, keep = _address(mv) try: - _check("cuMemcpyHtoD_v2", _load().cuMemcpyHtoD_v2(self.ptr + offset, addr, len(mv))) + with bound_context(self.device): + _check("cuMemcpyHtoD_v2", _load().cuMemcpyHtoD_v2(self.ptr + offset, addr, len(mv))) finally: del keep @@ -300,7 +339,8 @@ def copy_to_host(self, offset: int, out) -> None: raise RacerError("copy outside the allocation") addr, keep = _address(mv) try: - _check("cuMemcpyDtoH_v2", _load().cuMemcpyDtoH_v2(addr, self.ptr + offset, len(mv))) + with bound_context(self.device): + _check("cuMemcpyDtoH_v2", _load().cuMemcpyDtoH_v2(addr, self.ptr + offset, len(mv))) finally: del keep @@ -364,16 +404,20 @@ def close(self) -> None: self._keep = None -def host_register(buf) -> HostRegistration: - """Page-lock a writable host buffer with cuMemHostRegister.""" +def host_register(buf, device: int = 0) -> HostRegistration: + """Page-lock a writable host buffer with cuMemHostRegister. + + The registration is made in ``device``'s primary context (PORTABLE makes + it usable from every context); pass the device the buffer will feed. + """ lib = _load() - ensure_context(0) mv = memoryview(buf).cast("B") if mv.readonly: raise RacerError("buffer must be writable") arr = (ctypes.c_char * len(mv)).from_buffer(mv) addr = ctypes.addressof(arr) - _check("cuMemHostRegister_v2", lib.cuMemHostRegister_v2(addr, len(mv), CU_MEMHOSTREGISTER_PORTABLE)) + with bound_context(device): + _check("cuMemHostRegister_v2", lib.cuMemHostRegister_v2(addr, len(mv), CU_MEMHOSTREGISTER_PORTABLE)) # Drop the export so the owner can still close its mapping after unregistering. del arr mv.release() diff --git a/clients/python/racer/src/racer/host.py b/clients/python/racer/src/racer/host.py index 1621ae2d2..de24e51d0 100644 --- a/clients/python/racer/src/racer/host.py +++ b/clients/python/racer/src/racer/host.py @@ -42,14 +42,14 @@ def fileno(self) -> int: def pinned(self) -> bool: return self._pinned is not None - def pin(self) -> bool: - """Register the mapping with CUDA. Returns False when CUDA is missing.""" + def pin(self, device: int = 0) -> bool: + """Register the mapping with CUDA in ``device``'s context. Returns False when CUDA is missing.""" if self._pinned is not None: return True from . import cuda try: - self._pinned = cuda.host_register(self.view) + self._pinned = cuda.host_register(self.view, device) except cuda.CudaUnavailable: return False return True diff --git a/clients/python/racer/tests/test_cuda.py b/clients/python/racer/tests/test_cuda.py index 6319f6b3e..85b76da1b 100644 --- a/clients/python/racer/tests/test_cuda.py +++ b/clients/python/racer/tests/test_cuda.py @@ -49,3 +49,133 @@ def test_gpu_allocation_roundtrip(): fd = g.export_fd() assert fd >= 0 os.close(fd) + + +class _FakeLib: + """Records driver calls; contexts are ints, primary context of device d is 100 + d.""" + + def __init__(self, current: int = 0, fail: str | None = None) -> None: + self.current = current + self.fail = fail + self.calls: list[tuple] = [] + + def _rc(self, name: str) -> int: + return 1 if name == self.fail else 0 + + def cuDeviceGet(self, out, ordinal): + out._obj.value = ordinal + return 0 + + def cuDevicePrimaryCtxRetain(self, out, dev): + out._obj.value = 100 + dev + self.calls.append(("retain", dev)) + return 0 + + def cuCtxGetCurrent(self, out): + out._obj.value = self.current or None + return 0 + + def cuCtxSetCurrent(self, ctx): + value = ctx.value if isinstance(ctx, ctypes.c_void_p) else ctx + self.current = value or 0 + self.calls.append(("set", self.current)) + return 0 + + def cuMemcpyHtoD_v2(self, dst, src, n): + self.calls.append(("htod", self.current)) + return self._rc("cuMemcpyHtoD_v2") + + def cuMemcpyDtoH_v2(self, dst, src, n): + self.calls.append(("dtoh", self.current)) + return self._rc("cuMemcpyDtoH_v2") + + def cuMemHostRegister_v2(self, addr, n, flags): + self.calls.append(("register", self.current)) + return self._rc("cuMemHostRegister_v2") + + def cuMemHostUnregister(self, addr): + return 0 + + +@pytest.fixture +def fake(monkeypatch): + def make(**kw): + lib = _FakeLib(**kw) + monkeypatch.setattr(cuda, "_lib", lib) + monkeypatch.setattr(cuda, "_contexts", {}) + return lib + + return make + + +def _alloc(device: int) -> cuda.GpuAllocation: + # Skip __init__ (VMM calls); only the fields the copy paths use. + g = cuda.GpuAllocation.__new__(cuda.GpuAllocation) + g.device, g.size, g.ptr = device, 4096, 0x1000 + g._handle, g._reserved, g._mapped = None, False, False + return g + + +@pytest.mark.parametrize("current", [0, 100, 101]) +def test_copy_from_host_binds_allocation_context(fake, current): + lib = fake(current=current) + _alloc(2).copy_from_host(0, b"abc") + assert ("htod", 102) in lib.calls + assert lib.current == current + if current: + assert lib.calls[-1] == ("set", current) + + +def test_copy_from_host_restores_context_on_error(fake): + lib = fake(current=100, fail="cuMemcpyHtoD_v2") + with pytest.raises(cuda.CudaError): + _alloc(1).copy_from_host(0, b"abc") + assert lib.calls[-2:] == [("htod", 101), ("set", 100)] + assert lib.current == 100 + + +def test_copy_from_host_without_prior_context(fake): + lib = fake(current=0) + _alloc(1).copy_from_host(0, b"abc") + assert lib.calls == [("retain", 1), ("set", 101), ("htod", 101), ("set", 0)] + + +def test_copy_to_host_binds_allocation_context(fake): + lib = fake(current=100) + _alloc(3).copy_to_host(0, bytearray(3)) + assert ("dtoh", 103) in lib.calls and lib.current == 100 + + +def test_already_current_context_is_not_switched(fake): + lib = fake(current=101) + _alloc(1).copy_from_host(0, b"abc") + assert [c for c in lib.calls if c[0] == "set"] == [] + + +def test_host_register_uses_requested_device(fake): + lib = fake(current=0) + cuda.host_register(bytearray(64), 2).close() + assert ("register", 102) in lib.calls and lib.current == 0 + + +def test_host_register_restores_context_on_error(fake): + lib = fake(current=100, fail="cuMemHostRegister_v2") + with pytest.raises(cuda.CudaError): + cuda.host_register(bytearray(64), 1) + assert lib.current == 100 + + +def test_staging_pinned_for_destination_device(fake, tmp_path): + from racer.client import Client + + lib = fake(current=0) + c = Client.__new__(Client) + c._pool, c.staging_bytes, c.pin_staging = [], 4096, True + import threading + + c._pool_lock = threading.Lock() + buf = c._take_staging(3) + try: + assert ("register", 103) in lib.calls + finally: + buf.close() From 19e4615f34dc5a238b29df7119f7b7eec47886ef Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:38:11 +0000 Subject: [PATCH 49/87] fix(racer): cap descriptors received across a request head --- cmd/racer-dataplane/http/src/connection.rs | 14 ++++++- cmd/racer-dataplane/http/tests/exchanges.rs | 41 +++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/cmd/racer-dataplane/http/src/connection.rs b/cmd/racer-dataplane/http/src/connection.rs index c163f05be..16bba406d 100644 --- a/cmd/racer-dataplane/http/src/connection.rs +++ b/cmd/racer-dataplane/http/src/connection.rs @@ -321,6 +321,10 @@ pub struct HeadCompletion { pub _decoded: Option, } +/// Most descriptors retained across one request head. Excess descriptors are +/// closed on arrival and the head is marked truncated. +pub const MAX_HEAD_DESCRIPTORS: usize = 8; + /// Descriptors passed with `SCM_RIGHTS` while a request head was read. #[derive(Debug, Default)] pub struct ReceivedDescriptors { @@ -724,7 +728,15 @@ impl HttpIo { .await?; let mut sink = sink.borrow_mut(); sink.truncated |= received.truncated; - sink.descriptors.extend(received.descriptors); + for descriptor in received.descriptors { + if sink.descriptors.len() < MAX_HEAD_DESCRIPTORS { + sink.descriptors.push(descriptor); + } else { + // Over the aggregate cap: close now and reject the head. + drop(descriptor); + sink.truncated = true; + } + } Completion { buffer: received.buffer, bytes: received.bytes, diff --git a/cmd/racer-dataplane/http/tests/exchanges.rs b/cmd/racer-dataplane/http/tests/exchanges.rs index b72921133..153c52b96 100644 --- a/cmd/racer-dataplane/http/tests/exchanges.rs +++ b/cmd/racer-dataplane/http/tests/exchanges.rs @@ -768,3 +768,44 @@ fn request_heads_collect_passed_descriptors() { assert!(head.descriptors.descriptors.is_empty()); assert!(!head.descriptors.truncated); } + +/// Descriptors retained across all fragments of one head are capped; excess +/// descriptors are closed immediately and the head is marked truncated. +#[test] +fn request_head_descriptors_are_capped_across_fragments() { + use std::io::{Read, Write}; + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + let io = io(); + let (mut client, server) = UnixStream::pair().unwrap(); + let server = ConnectionLease::from_reserved(server.into(), (), ()).unwrap(); + let mut fds = [0; 2]; + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + let mut read = unsafe { std::fs::File::from_raw_fd(fds[0]) }; + let write = unsafe { OwnedFd::from_raw_fd(fds[1]) }; + let fragments = http1::connection::MAX_HEAD_DESCRIPTORS + 4; + send_with_fds(&client, b"POST /x HTTP/1.1\r\n", &[write.as_raw_fd()]); + for _ in 1..fragments { + send_with_fds( + &client, + b"X: y\r\n", + &[write.as_raw_fd(), write.as_raw_fd()], + ); + } + drop(write); + client.write_all(b"Content-Length: 0\r\n\r\n").unwrap(); + let head = drive( + &io, + io.receive_request_head_with_descriptors(server, &RequestScope, 4096), + ) + .unwrap() + .value + .unwrap(); + assert!(head.descriptors.truncated); + assert_eq!( + head.descriptors.descriptors.len(), + http1::connection::MAX_HEAD_DESCRIPTORS + ); + drop(head); + let mut byte = [0]; + assert_eq!(read.read(&mut byte).unwrap(), 0); +} From 394b6bb42fd2ad17b53a26a2411765f3c89c619d Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:50:48 +0000 Subject: [PATCH 50/87] fix(racer): close vLLM weights iterator before closing the client --- .../racer-vllm/src/racer_vllm/loader.py | 8 ++- .../python/racer-vllm/tests/test_loader.py | 57 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/clients/python/racer-vllm/src/racer_vllm/loader.py b/clients/python/racer-vllm/src/racer_vllm/loader.py index 6a21ade40..58a08a558 100644 --- a/clients/python/racer-vllm/src/racer_vllm/loader.py +++ b/clients/python/racer-vllm/src/racer_vllm/loader.py @@ -70,7 +70,13 @@ def load_weights(self, model, model_config) -> None: with client: backend = TorchBackend(clone=s["clone"], use_vmm=s["use_vmm"]) weights = iter_weights(client, uri, backend, s["staging_bytes"], s["concurrency"]) - loaded = model.load_weights(weights) + try: + loaded = model.load_weights(weights) + finally: + # Run the generator's cleanup (executor shutdown and GPU + # fences) before the client closes, even if load_weights + # raised or returned without exhausting the iterator. + weights.close() if loaded is not None and getattr(model_config, "quantization", None) is None: missing = {n for n, _ in model.named_parameters()} - set(loaded) if missing: diff --git a/clients/python/racer-vllm/tests/test_loader.py b/clients/python/racer-vllm/tests/test_loader.py index e3508467a..866c1ed24 100644 --- a/clients/python/racer-vllm/tests/test_loader.py +++ b/clients/python/racer-vllm/tests/test_loader.py @@ -311,3 +311,60 @@ def test_iter_weights_fences_on_exception(server, uri): with pytest.raises(RuntimeError): gen.throw(RuntimeError("consumer failed")) b.assert_fenced() + + +@pytest.mark.parametrize("mode", ["raise", "early"]) +def test_load_weights_closes_iterator_before_client(server, uri, monkeypatch, tmp_path, mode): + registry = install_fake_vllm(monkeypatch) + loader.register() + setup_model(server, uri) + model.fetch(uri, str(tmp_path / "m"), client=racer.Client(socket_path=server.path)) + monkeypatch.setenv("RACER_SOCKET", server.path) + monkeypatch.setattr(loader, "TorchBackend", lambda **kw: MemoryBackend()) + events = [] + real_iter = loader.iter_weights + + def tracking_iter(*a, **kw): + try: + yield from real_iter(*a, **kw) + finally: + events.append("weights closed") + + monkeypatch.setattr(loader, "iter_weights", tracking_iter) + real_client_for = loader.client_for + + def tracking_client_for(*a, **kw): + c = real_client_for(*a, **kw) + inner_exit = c.__exit__ + + class Wrap: + def __getattr__(self, n): + return getattr(c, n) + + def __enter__(self): + c.__enter__() + return self + + def __exit__(self, *exc): + events.append("client closed") + return inner_exit(*exc) + + return Wrap() + + monkeypatch.setattr(loader, "client_for", tracking_client_for) + inst = registry["racer"](types.SimpleNamespace(model_loader_extra_config={"staging_bytes": 400})) + mc = types.SimpleNamespace(model=str(tmp_path / "m"), quantization=None) + + class Model: + def load_weights(self, weights): + next(weights) + if mode == "raise": + raise RuntimeError("boom") + return None + + if mode == "raise": + with pytest.raises(RuntimeError, match="boom"): + inst.load_weights(Model(), mc) + else: + inst.load_weights(Model(), mc) + assert events == ["weights closed", "client closed"] From 0dd054482bfa539fd0066ffa8f7040274ab2fbeb Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:50:55 +0000 Subject: [PATCH 51/87] fix(racer): release dedicated GPU targets as soon as they are consumed --- .../racer-vllm/src/racer_vllm/weights.py | 16 ++++++---- .../python/racer-vllm/tests/test_loader.py | 29 +++++++++++++++++++ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/clients/python/racer-vllm/src/racer_vllm/weights.py b/clients/python/racer-vllm/src/racer_vllm/weights.py index 5c56b62fd..767c68e4a 100644 --- a/clients/python/racer-vllm/src/racer_vllm/weights.py +++ b/clients/python/racer-vllm/src/racer_vllm/weights.py @@ -69,7 +69,9 @@ def iter_weights( queued GPU work that reads it has finished. Every staging and dedicated target is fenced with done() and wait() before the pipeline drops its reference, including on early close or error, so external allocations - are never released while queued GPU work may still read them. + are never released while queued GPU work may still read them. Dedicated + targets are fenced and dropped as soon as the consumer resumes past + them, so oversized tensors do not accumulate in GPU memory. """ if depth < 1: raise ValueError("depth must be at least 1") @@ -97,8 +99,6 @@ def fill() -> bool: inflight.append((pool.submit(read, req, etag, chunk, target), chunk)) return True - # Dedicated targets the consumer has moved past; fenced at the end. - retired: list = [] active = None try: while fill(): @@ -110,7 +110,10 @@ def fill() -> bool: active = None backend.done(target) if chunk.dedicated: - retired.append(target) + # Fence and drop now so oversized targets do not accumulate; + # tensors the consumer kept hold their own storage reference. + backend.wait(target) + del target else: free.append(target) finally: @@ -119,10 +122,11 @@ def fill() -> bool: pool.shutdown(wait=True) if active is not None: backend.done(active) - retired.append(active) + backend.wait(active) + active = None # Reads have stopped; fence consumer GPU work before release. Targets # still in flight were fenced by read() and the consumer never saw them. - for target in retired + free: + for target in free: backend.wait(target) diff --git a/clients/python/racer-vllm/tests/test_loader.py b/clients/python/racer-vllm/tests/test_loader.py index 866c1ed24..463a38c1f 100644 --- a/clients/python/racer-vllm/tests/test_loader.py +++ b/clients/python/racer-vllm/tests/test_loader.py @@ -368,3 +368,32 @@ def load_weights(self, weights): else: inst.load_weights(Model(), mc) assert events == ["weights closed", "client closed"] + + +def test_iter_weights_releases_dedicated_when_consumed(server, uri): + import gc + import weakref + + setup_model(server, uri) + c = racer.Client(socket_path=server.path, staging_bytes=4096) + refs = [] + + class _WeakBackend(_FencingBackend): + def new_dedicated(self, size): + t = self.Target(size) + refs.append(weakref.ref(t)) + return t + + b = _WeakBackend() + gen = iter_weights(c, uri, b, staging_bytes=400, depth=2) + for name, _ in gen: + if name == "w3": + break + assert len(refs) == 1 and refs[0]() is not None + target_id = id(refs[0]()) + # Resuming past w3 fences the dedicated target and drops it. + next(gen, None) + assert target_id in b.waited + gc.collect() + assert refs[0]() is None + gen.close() From 15d57bc1eefb4d63dc7c46d8c60c34bac861555a Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:54:26 +0000 Subject: [PATCH 52/87] fix(racer): retain failed GPU verdicts the owner mailbox cannot carry --- cmd/racer-dataplane/src/read/dispatch.rs | 162 ++++++++++++++++++++++- cmd/racer-dataplane/src/read/sealed.rs | 1 + 2 files changed, 158 insertions(+), 5 deletions(-) diff --git a/cmd/racer-dataplane/src/read/dispatch.rs b/cmd/racer-dataplane/src/read/dispatch.rs index 8de785af8..27dac7366 100644 --- a/cmd/racer-dataplane/src/read/dispatch.rs +++ b/cmd/racer-dataplane/src/read/dispatch.rs @@ -62,6 +62,37 @@ thread_local! { struct Mailbox { worker: WorkerId, inner: Arc, + /// Failed GPU verdicts the mailbox could not carry. The owning endpoint + /// applies them on its next poll so a corrupt copy is never left eligible. + failed_verdicts: std::sync::Mutex>, +} + +/// Bound on retained failed verdicts per worker. Each entry is one failed GPU +/// verification; beyond this the owner is far behind and the oldest are kept. +const FAILED_VERDICT_LIMIT: usize = 4096; + +impl Mailbox { + /// Retain a failed verdict for the owner. Returns false only when full. + fn retain_failed_verdict(&self, report: super::sealed::SealedReport) -> bool { + let mut pending = self + .failed_verdicts + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + if pending.len() >= FAILED_VERDICT_LIMIT { + return false; + } + pending.push(report); + true + } + + fn take_failed_verdicts(&self) -> Vec { + std::mem::take( + &mut *self + .failed_verdicts + .lock() + .unwrap_or_else(|poison| poison.into_inner()), + ) + } } impl std::ops::Deref for Mailbox { type Target = Handoff; @@ -376,6 +407,7 @@ impl WorkerDirectory { Arc::new(Mailbox { worker, inner: Arc::new(Handoff::new(capacity).expect("validated capacity")), + failed_verdicts: std::sync::Mutex::new(Vec::new()), }) }) .collect(), @@ -932,21 +964,35 @@ impl WorkerDirectory { } return; } + // A failed verdict must reach the owner, or the corrupt copy stays + // eligible. When the driver queue or mailbox cannot take it, retain + // it on the owner's mailbox for the owner's next poll. + let retain = |report: super::sealed::SealedReport| { + if !passed && let Ok(mailbox) = self.mailbox(owner) { + mailbox.retain_failed_verdict(report); + } + }; let Ok(permit) = uring_runtime::drivers::reserve() else { + retain(report); return; }; // The request may already be ending; the verdict must still arrive. let deadline = uring_runtime::environment::now() + std::time::Duration::from_secs(5); let Ok(detached) = RequestScope::new(scope.request, deadline) else { + retain(report); return; }; - if let Ok(receipt) = - self.submit(owner, Work::SealedVerdict(report, passed), &detached, None) - { - permit.submit_detached(Box::pin(async move { + let kept = (!passed).then(|| report.clone()); + match self.submit(owner, Work::SealedVerdict(report, passed), &detached, None) { + Ok(receipt) => permit.submit_detached(Box::pin(async move { let _ = receipt.await; Ok::<(), Error>(()) - })); + })), + Err(_) => { + if let Some(report) = kept { + retain(report); + } + } } } @@ -1072,6 +1118,12 @@ impl WorkerEndpoint { self.directory.simulation_crash(); self.active.simulation_crash(); } + /// Apply failed GPU verdicts other workers could not send through the mailbox. + fn apply_failed_verdicts(&self) { + for report in self.mailbox.take_failed_verdicts() { + self.local.fill.sealed_verdict(report, false); + } + } pub fn stop_admission(&self) { self.directory.warm.stop(); self.mailbox.stop_admission(); @@ -1084,6 +1136,7 @@ impl WorkerEndpoint { pub fn drain<'a>(&'a mut self, scope: &'a RequestScope) -> Operation<'a, ()> { self.stop_admission(); Box::pin(std::future::poll_fn(move |cx| { + self.apply_failed_verdicts(); self.directory.warm.poll_releases( self.mailbox.worker, self.local.fill.dependencies.writer.index(), @@ -1104,6 +1157,7 @@ impl WorkerEndpoint { } /// Register a reactor waker and execute at most `work_budget` command/poll steps. pub fn poll(&mut self, cx: &mut Context<'_>, work_budget: usize) -> Result<()> { + self.apply_failed_verdicts(); self.directory.warm.poll_deadlines(); self.directory.warm.poll_releases( self.mailbox.worker, @@ -1562,6 +1616,104 @@ mod tests { } job.stop(); } + /// A page owned by `worker` in the two-worker test map. + fn page_owned_by(directory: &WorkerDirectory, worker: WorkerId) -> PageId { + (0..1024) + .map(|number| PageId { + version: version(), + number: crate::model::PageNumber(number), + }) + .find(|page| directory.page_owner(page).unwrap() == worker) + .unwrap() + } + fn failed_report(page: PageId) -> super::super::sealed::SealedReport { + super::super::sealed::SealedReport { + page, + copy: None, + token: None, + } + } + /// Deliver retained verdicts through the owner's endpoint and count them. + fn applied_by_owner(directory: &Arc) -> u64 { + let (owner, _fixture) = crate::read::tests::admission_coordinator(); + let before = owner + .fill + .metrics + .count(crate::telemetry::Event::CorruptMiss); + let mut endpoint = directory.install(WorkerId(1), owner.clone()).unwrap(); + endpoint.poll_budgeted(0).unwrap(); + let applied = owner + .fill + .metrics + .count(crate::telemetry::Event::CorruptMiss) + - before; + drop(endpoint); + applied + } + #[test] + fn failed_sealed_verdict_survives_full_driver_queue() { + let queue = Rc::new(uring_runtime::drivers::DriverQueue::new(1)); + let _queue = queue.enter(); + let workers = vec![WorkerId(0), WorkerId(1)]; + let directory = Arc::new( + WorkerDirectory::new( + Arc::new(WorkerMap::new(workers.clone()).unwrap()), + workers, + 4, + ) + .unwrap(), + ); + let page = page_owned_by(&directory, WorkerId(1)); + let full = uring_runtime::drivers::reserve().unwrap(); + { + let (local, _fixture) = crate::read::tests::admission_coordinator(); + let _local = directory.simulation_scope(Some((WorkerId(0), local))); + directory.report_sealed(failed_report(page.clone()), false, &scope()); + // A passing verdict is only a persistence hint and is not retained. + directory.report_sealed(failed_report(page), true, &scope()); + } + drop(full); + assert_eq!(applied_by_owner(&directory), 1); + assert!(directory.mailboxes[1].take_failed_verdicts().is_empty()); + } + #[test] + fn failed_sealed_verdict_survives_full_mailbox() { + let queue = Rc::new(uring_runtime::drivers::DriverQueue::new(64)); + let _queue = queue.enter(); + let workers = vec![WorkerId(0), WorkerId(1)]; + let directory = Arc::new( + WorkerDirectory::new( + Arc::new(WorkerMap::new(workers.clone()).unwrap()), + workers, + 1, + ) + .unwrap(), + ); + directory.mailboxes[1].install().unwrap(); + let page = page_owned_by(&directory, WorkerId(1)); + { + let (local, _fixture) = crate::read::tests::admission_coordinator(); + let _local = directory.simulation_scope(Some((WorkerId(0), local))); + // The first verdict takes the only mailbox slot; the second is retained. + directory.report_sealed(failed_report(page.clone()), false, &scope()); + assert_eq!(directory.mailboxes[1].outstanding(), 1); + directory.report_sealed(failed_report(page), false, &scope()); + } + assert_eq!(directory.mailboxes[1].take_failed_verdicts().len(), 1); + } + #[test] + fn failed_sealed_verdicts_are_bounded() { + let directory = directory(1); + let page = page_owned_by(&directory, WorkerId(1)); + for _ in 0..FAILED_VERDICT_LIMIT { + assert!(directory.mailboxes[1].retain_failed_verdict(failed_report(page.clone()))); + } + assert!(!directory.mailboxes[1].retain_failed_verdict(failed_report(page))); + assert_eq!( + directory.mailboxes[1].take_failed_verdicts().len(), + FAILED_VERDICT_LIMIT + ); + } #[test] fn canonical_order_rejects_empty_and_duplicate_identities() { assert!(matches!( diff --git a/cmd/racer-dataplane/src/read/sealed.rs b/cmd/racer-dataplane/src/read/sealed.rs index a5818255c..8f2e42aa7 100644 --- a/cmd/racer-dataplane/src/read/sealed.rs +++ b/cmd/racer-dataplane/src/read/sealed.rs @@ -86,6 +86,7 @@ pub struct SealedPage { } /// What a page's owning worker needs to act on the GPU's verdict. +#[derive(Clone)] pub struct SealedReport { pub(crate) page: PageId, From bbae30cab1c600af36e99b4d32ae6e86ab61df6c Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:52:11 +0000 Subject: [PATCH 53/87] fix(racer): ship cuFile library in CUDA dataplane image --- docs/content/guides/racer-vllm.md | 5 +++++ images/racer-dataplane/Containerfile | 33 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/docs/content/guides/racer-vllm.md b/docs/content/guides/racer-vllm.md index 4c7f90645..ff61ba2ee 100644 --- a/docs/content/guides/racer-vllm.md +++ b/docs/content/guides/racer-vllm.md @@ -32,6 +32,11 @@ podman build --build-arg RACER_NATIVE_CUDA=true \ -t /racer-dataplane: -f images/racer-dataplane/Containerfile . ``` +The CUDA image also installs NVIDIA's cuFile library (`libcufile-12-9`) for +GPUDirect Storage reads, because the NVIDIA container runtime injects only the +driver libraries. cuFile uses its packaged default configuration; without +GPUDirect Storage support on the host, disk reads fall back to host memory. + GPU mode is `auto` by default. To make the dataplane fail when it finds no GPU, set it to `true`: diff --git a/images/racer-dataplane/Containerfile b/images/racer-dataplane/Containerfile index 72a6a1dd4..3191578a3 100644 --- a/images/racer-dataplane/Containerfile +++ b/images/racer-dataplane/Containerfile @@ -103,6 +103,39 @@ RUN set -eu; \ # symbol versions, so the binary and verbs adapter run on noble's newer ones. FROM ${RUNTIME_IMAGE} ARG RACER_NATIVE_RDMA=true +ARG RACER_NATIVE_CUDA=false +# cuFile user-space library for GPUDirect Storage. The NVIDIA container runtime +# injects driver libraries only, not the toolkit's libcufile.so.0, so the CUDA +# image ships it. The repo matches RUNTIME_IMAGE (ubuntu:noble). +ARG CUDA_CUFILE_PACKAGE=libcufile-12-9 +ARG CUDA_RUNTIME_REPO=ubuntu2404 +RUN set -eu; \ + case "$RACER_NATIVE_CUDA" in \ + true) \ + case "$(uname -m)" in \ + x86_64) repo=x86_64 ;; \ + aarch64) repo=sbsa ;; \ + *) echo "unsupported CUDA architecture $(uname -m)" >&2; exit 1 ;; \ + esac; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates curl; \ + curl -fsSL -o /tmp/cuda-keyring.deb \ + "https://developer.download.nvidia.com/compute/cuda/repos/$CUDA_RUNTIME_REPO/$repo/cuda-keyring_1.1-1_all.deb"; \ + dpkg -i /tmp/cuda-keyring.deb; \ + rm /tmp/cuda-keyring.deb; \ + apt-get update; \ + apt-get install -y --no-install-recommends "$CUDA_CUFILE_PACKAGE"; \ + apt-get purge -y --auto-remove curl; \ + lib=$(dirname "$(find /usr/local -name 'libcufile.so.0' | head -n 1)"); \ + test -n "$lib" && test "$lib" != .; \ + echo "$lib" > /etc/ld.so.conf.d/racer-cufile.conf; \ + ldconfig; \ + ldconfig -p | grep -q 'libcufile.so.0'; \ + apt-get clean; \ + rm -rf /var/lib/apt/lists/* ;; \ + false) ;; \ + *) echo "RACER_NATIVE_CUDA must be true or false" >&2; exit 1 ;; \ + esac RUN set -eu; \ case "$RACER_NATIVE_RDMA" in \ true) set -- libibverbs1 ibverbs-providers ;; \ From 71450b03f3324f7446c78febca5ea401a80466c6 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:30:46 +0000 Subject: [PATCH 54/87] feat(racer): add dma-buf device page pools to rdma-verbs Add ABI version 5 of the verbs adapter with rdma_verbs_register_dmabuf (ibv_reg_dmabuf_mr). Client::attach_device registers caller-owned dma-buf memory as a rail's device pool, and Client::grant_device binds a window over a device buffer. After the same fence as host grants, the arrival is Arrival::Device with a DevicePage that holds its buffer until dropped. Device regions have no CPU mapping: sends, writes, and receives reject them in Rust, the C adapter, and the simulation, which maps a memfd in place of the dma-buf. --- cmd/racer-dataplane/src/rdma/links.rs | 3 +- cmd/racer-dataplane/src/rdma/links/tests.rs | 8 +- cmd/racer-dataplane/verbs/native/verbs.c | 43 ++- cmd/racer-dataplane/verbs/src/ffi.rs | 101 ++++++- cmd/racer-dataplane/verbs/src/lib.rs | 33 ++- cmd/racer-dataplane/verbs/src/link.rs | 80 +++++- cmd/racer-dataplane/verbs/src/link/tests.rs | 8 +- cmd/racer-dataplane/verbs/src/node.rs | 104 +++++++- cmd/racer-dataplane/verbs/src/pool.rs | 273 ++++++++++++++++++- cmd/racer-dataplane/verbs/src/simulation.rs | 279 +++++++++++++++++++- cmd/racer-dataplane/verbs/tests/links.rs | 170 +++++++++++- 11 files changed, 1051 insertions(+), 51 deletions(-) diff --git a/cmd/racer-dataplane/src/rdma/links.rs b/cmd/racer-dataplane/src/rdma/links.rs index 75484386d..e2a1d4f9f 100644 --- a/cmd/racer-dataplane/src/rdma/links.rs +++ b/cmd/racer-dataplane/src/rdma/links.rs @@ -853,7 +853,8 @@ impl PendingReceive { .await; let page = match arrival { Ok(Arrival::Page(page)) => page, - Ok(Arrival::Released) => return Err(Error::Io), + // Only `grant` reaches here, and it never asks for a device page. + Ok(Arrival::Released | Arrival::Device(_)) => return Err(Error::Io), Err(error @ Error::Cancelled) => return Err(error), Err(_) => { scope.check()?; diff --git a/cmd/racer-dataplane/src/rdma/links/tests.rs b/cmd/racer-dataplane/src/rdma/links/tests.rs index 930a141e8..22761751c 100644 --- a/cmd/racer-dataplane/src/rdma/links/tests.rs +++ b/cmd/racer-dataplane/src/rdma/links/tests.rs @@ -166,7 +166,7 @@ pub(crate) fn receive_registered( drive!(w, w.b.links.send(&a, &grant, page, &scope)).unwrap(); match drive!(w, poll_fn(|cx| pending.arrival.poll(cx))).unwrap() { Arrival::Page(page) => page, - Arrival::Released => panic!("expected a page"), + Arrival::Released | Arrival::Device(_) => panic!("expected a page"), } } @@ -382,7 +382,7 @@ fn setup_page_both_directions_and_release() { let arrival = drive!(w, poll_fn(|cx| pending.arrival.poll(cx))).unwrap(); match arrival { Arrival::Page(page) => assert_eq!(page.bytes(), &[0x5a; 35]), - Arrival::Released => panic!("expected a page"), + Arrival::Released | Arrival::Device(_) => panic!("expected a page"), } // Release path: the receiver sees `Released` and the buffer is free. let pending = drive!(w, receiver.grant(from, RAIL, TransferId([4; 16]), &scope)).unwrap(); @@ -634,7 +634,7 @@ mod send_failures { fn arrived(w: &mut World, pending: &PendingReceive) -> Vec { match drive!(w, poll_fn(|cx| pending.arrival.poll(cx))).unwrap() { Arrival::Page(page) => page.bytes().to_vec(), - Arrival::Released => panic!("expected a page"), + Arrival::Released | Arrival::Device(_) => panic!("expected a page"), } } @@ -796,7 +796,7 @@ mod send_in_place { let arrival = drive!(w, poll_fn(|cx| pending.arrival.poll(cx))).unwrap(); match arrival { Arrival::Page(page) => assert_eq!(page.bytes(), &[0xc3; 35][..]), - Arrival::Released => panic!("expected a page"), + Arrival::Released | Arrival::Device(_) => panic!("expected a page"), } pump!(w); assert!( diff --git a/cmd/racer-dataplane/verbs/native/verbs.c b/cmd/racer-dataplane/verbs/native/verbs.c index edb87e16c..1639eb02d 100644 --- a/cmd/racer-dataplane/verbs/native/verbs.c +++ b/cmd/racer-dataplane/verbs/native/verbs.c @@ -64,10 +64,14 @@ _Static_assert(IBV_WC_SEND == 0 && IBV_WC_RDMA_WRITE == 1 && IBV_WC_BIND_MW == 5 /* Private handles. Rust only holds pointers to these. */ struct rdma_verbs_device { struct ibv_context *ctx; struct ibv_pd *pd; }; struct rdma_verbs_qp { struct ibv_qp *qp; struct ibv_cq *cq; }; -struct rdma_verbs_mr { struct ibv_mr *mr; void *bytes; size_t length; }; +/* `bytes` is NULL for an implicit or dma-buf region. `addr` is the address + * windows bind from: the buffer pointer, or the dma-buf iova. `device` is set + * for a dma-buf region, which only receives through windows. */ +struct rdma_verbs_mr { struct ibv_mr *mr; void *bytes; size_t length; uint64_t addr; int device; }; -/* ABI version. Rust refuses to load any other value. */ -uint32_t rdma_verbs_abi(void) { return 4; } +/* ABI version. Rust refuses to load any other value. + * Version 5 added rdma_verbs_register_dmabuf. */ +uint32_t rdma_verbs_abi(void) { return 5; } /* Distinct from negative errno: cleanup leaked a resource. Rust must keep * the admission owner and adapter loaded. Stop before another resource leaks. */ @@ -246,11 +250,30 @@ void *rdma_verbs_register(struct rdma_verbs_device *d, uint64_t length, uint32_t if (!m) return NULL; if (posix_memalign(&m->bytes, 4096, (size_t)length)) { free(m); return NULL; } memset(m->bytes, 0, (size_t)length); m->length = (size_t)length; + m->addr = (uintptr_t)m->bytes; m->mr = ibv_reg_mr(d->pd, m->bytes, (size_t)length, flags); if (!m->mr) { free(m->bytes); free(m); return NULL; } return m; } -/* Deregister and free the buffer. */ +/* Register `length` bytes of a dma-buf, starting at `offset` in it, as page + * buffers (LOCAL_WRITE | MW_BIND, so peers write only through windows). + * Windows over it bind from `iova`. The caller keeps the memory behind the + * dma-buf alive until deregister succeeds. The kernel holds its own reference + * to the dma-buf, so `fd` may be closed after this returns. The CPU cannot + * reach the memory: rdma_verbs_bytes returns NULL, and send, write, and + * receive posts reject the region. Needs rdma-core 34 or later. */ +void *rdma_verbs_register_dmabuf(struct rdma_verbs_device *d, uint64_t offset, uint64_t length, + uint64_t iova, int fd) { + if (!length || length > SIZE_MAX || fd < 0) return NULL; + struct rdma_verbs_mr *m = calloc(1, sizeof(*m)); + if (!m) return NULL; + m->length = (size_t)length; m->addr = iova; m->device = 1; + m->mr = ibv_reg_dmabuf_mr(d->pd, offset, (size_t)length, iova, fd, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_MW_BIND); + if (!m->mr) { free(m); return NULL; } + return m; +} +/* Deregister and free the buffer, if this side allocated one. */ int rdma_verbs_deregister(struct rdma_verbs_mr *m) { int rc = ibv_dereg_mr(m->mr); if (!rc) { free(m->bytes); free(m); } @@ -279,7 +302,7 @@ void *rdma_verbs_register_implicit(struct rdma_verbs_device *d) { m->bytes = NULL; m->length = SIZE_MAX; return m; } -/* The buffer behind a registered region. */ +/* The buffer behind a registered region. NULL for an implicit or dma-buf region. */ void *rdma_verbs_bytes(struct rdma_verbs_mr *m) { return m->bytes; } /* Allocate a type 2 memory window. Writes the key to use for its next bind. */ void *rdma_verbs_window(struct rdma_verbs_device *d, uint32_t *key) { @@ -289,6 +312,8 @@ void *rdma_verbs_window(struct rdma_verbs_device *d, uint32_t *key) { } /* Free a memory window. */ int rdma_verbs_window_free(struct ibv_mw *mw) { return ibv_dealloc_mw(mw); } +/* True if `m` has CPU-reachable or implicit memory a post may use as an SGE. */ +static int host_sge(const struct rdma_verbs_mr *m) { return !m->device; } /* True if [offset, offset + length) is a nonempty range inside `m`. */ static int in_region(const struct rdma_verbs_mr *m, uint64_t offset, uint32_t length) { return length && offset <= m->length && length <= m->length - offset; @@ -306,7 +331,7 @@ int rdma_verbs_bind(struct rdma_verbs_qp *q, struct ibv_mw *mw, struct rdma_verb wr.send_flags = IBV_SEND_SIGNALED | IBV_SEND_FENCE; wr.bind_mw.mw = mw; wr.bind_mw.rkey = key; wr.bind_mw.bind_info.mr = m->mr; - wr.bind_mw.bind_info.addr = (uintptr_t)m->bytes + offset; + wr.bind_mw.bind_info.addr = m->addr + offset; wr.bind_mw.bind_info.length = length; wr.bind_mw.bind_info.mw_access_flags = IBV_ACCESS_REMOTE_WRITE; return ibv_post_send(q->qp, &wr, &bad); @@ -325,7 +350,7 @@ int rdma_verbs_invalidate(struct rdma_verbs_qp *q, uint32_t key, uint64_t id) { int rdma_verbs_write_imm(struct rdma_verbs_qp *q, struct rdma_verbs_mr *m, uint64_t offset, uint32_t length, uint64_t address, uint32_t key, uint32_t imm, uint64_t id) { - if (!in_region(m, offset, length)) return EINVAL; + if (!host_sge(m) || !in_region(m, offset, length)) return EINVAL; struct ibv_sge sge = { .addr = (uintptr_t)m->bytes + offset, .length = length, .lkey = m->mr->lkey }; struct ibv_send_wr wr = {0}, *bad = NULL; @@ -338,7 +363,7 @@ int rdma_verbs_write_imm(struct rdma_verbs_qp *q, struct rdma_verbs_mr *m, uint6 * the peer's posted receives. */ int rdma_verbs_send(struct rdma_verbs_qp *q, struct rdma_verbs_mr *m, uint64_t offset, uint32_t length, uint64_t id) { - if (!in_region(m, offset, length)) return EINVAL; + if (!host_sge(m) || !in_region(m, offset, length)) return EINVAL; struct ibv_sge sge = { .addr = (uintptr_t)m->bytes + offset, .length = length, .lkey = m->mr->lkey }; struct ibv_send_wr wr = {0}, *bad = NULL; @@ -349,7 +374,7 @@ int rdma_verbs_send(struct rdma_verbs_qp *q, struct rdma_verbs_mr *m, uint64_t o /* Post a receive into `length` bytes of `m` at `offset`. */ int rdma_verbs_post_recv(struct rdma_verbs_qp *q, struct rdma_verbs_mr *m, uint64_t offset, uint32_t length, uint64_t id) { - if (!in_region(m, offset, length)) return EINVAL; + if (!host_sge(m) || !in_region(m, offset, length)) return EINVAL; struct ibv_sge sge = { .addr = (uintptr_t)m->bytes + offset, .length = length, .lkey = m->mr->lkey }; struct ibv_recv_wr wr = {0}, *bad = NULL; diff --git a/cmd/racer-dataplane/verbs/src/ffi.rs b/cmd/racer-dataplane/verbs/src/ffi.rs index 4927f9ac9..018cf96f7 100644 --- a/cmd/racer-dataplane/verbs/src/ffi.rs +++ b/cmd/racer-dataplane/verbs/src/ffi.rs @@ -3,7 +3,7 @@ //! Only the native thread uses these. Each type owns one RDMA object and frees //! it on drop. If freeing fails, the object is leaked on purpose, along with //! everything it depends on, because the NIC may still be using it. -use crate::{Error, GuardOwner, Result}; +use crate::{DeviceMemory, Error, GuardOwner, Result}; use std::{ cell::{Cell, RefCell}, @@ -109,8 +109,8 @@ struct Completion { const WC_IMM: u32 = 1; /// Native adapter ABI version this crate speaks. -#[cfg(all(feature = "native", target_os = "linux"))] -const ABI_VERSION: u32 = 4; +#[cfg(any(test, all(feature = "native", target_os = "linux")))] +const ABI_VERSION: u32 = 5; /// Completion opcodes (`enum ibv_wc_opcode`), pinned by C static asserts. pub(crate) mod opcode { @@ -166,6 +166,10 @@ impl Access { /// Never passed to `rdma_verbs_register`; only a local write source. pub(crate) const IMPLICIT: Self = Self(8); + /// Marks a dma-buf page region from `rdma_verbs_register_dmabuf`. Never + /// passed to `rdma_verbs_register`; peers write it through windows only. + pub(crate) const DEVICE: Self = Self(1 | 16); + /// The raw ABI value. pub(crate) fn bits(self) -> u32 { self.0 @@ -257,6 +261,8 @@ native_api! { register_implicit: unsafe extern "C" fn(*mut c_void) -> *mut c_void, + register_dmabuf: unsafe extern "C" fn(*mut c_void, u64, u64, u64, c_int) -> *mut c_void, + deregister: unsafe extern "C" fn(*mut c_void) -> c_int, bytes: unsafe extern "C" fn(*mut c_void) -> *mut u8, @@ -489,11 +495,18 @@ fn discover_with_api( /// /// Posts do not track which ranges the NIC may touch; their caller must. /// Holds the node's guard and leaks it if deregister fails. +/// +/// A dma-buf region (`memory` is set) has no CPU mapping. It can only be +/// bound to windows for receives; sends, writes, and receives reject it. pub(crate) struct NativeRegion { device: Rc, raw: NonNull, + /// The dma-buf owner. Kept alive while registered, and leaked with the + /// region if deregister fails. + memory: Option, + /// Allocated size. length: usize, @@ -520,12 +533,56 @@ impl NativeRegion { Ok(Rc::new(Self { device, raw, + memory: None, length, quota: Some(quota), access, })) } + /// Register all of `memory`'s dma-buf range as page buffers. `memory` + /// stays alive until deregister succeeds, or forever if it fails. + pub(crate) fn dmabuf( + device: Rc, + memory: DeviceMemory, + quota: Arc, + ) -> Result> { + use std::os::fd::AsRawFd; + + let length = memory.length(); + if length == 0 || u64::try_from(length).is_err() { + return Err(Error::InvalidRange); + } + let raw = NonNull::new(unsafe { + (device.api.register_dmabuf)( + device.raw.as_ptr(), + memory.offset(), + length as u64, + memory.iova(), + memory.fd().as_raw_fd(), + ) + }) + .ok_or(Error::Unavailable)?; + Ok(Rc::new(Self { + device, + raw, + memory: Some(memory), + length, + quota: Some(quota), + access: Access::DEVICE, + })) + } + + /// The dma-buf owner, if this is a dma-buf region. + pub(crate) fn memory(&self) -> Option<&DeviceMemory> { + self.memory.as_ref() + } + + /// True for a dma-buf region from [`Self::dmabuf`]. + pub(crate) fn is_device(&self) -> bool { + self.memory.is_some() + } + /// Register the whole process address space as one implicit on-demand /// paging MR, for zero-copy write sources. `None` if the device lacks /// implicit ODP for RC writes. @@ -538,6 +595,7 @@ impl NativeRegion { Some(Rc::new(Self { device, raw, + memory: None, length: usize::MAX, quota: Some(quota), access: Access::IMPLICIT, @@ -552,7 +610,11 @@ impl NativeRegion { /// The start of the registered buffer, valid while `self` lives. /// /// The caller must not read or write any range the NIC may still access. + /// Null for a dma-buf region. pub(crate) fn as_ptr(&self) -> *mut u8 { + if self.is_device() { + return std::ptr::null_mut(); + } unsafe { (self.device.api.bytes)(self.raw.as_ptr()) } } @@ -566,6 +628,9 @@ impl NativeRegion { /// The address peers use to write here. pub(crate) fn address(&self) -> u64 { + if let Some(memory) = &self.memory { + return memory.iova(); + } #[cfg(any(test, feature = "simulation"))] if simulation::is_api(&self.device.api) { return simulation::address(self.raw); @@ -580,10 +645,11 @@ impl NativeRegion { } impl Drop for NativeRegion { - /// Deregister. On failure, leak the device and guard. + /// Deregister. On failure, leak the device, guard, and dma-buf owner. fn drop(&mut self) { if unsafe { (self.device.api.deregister)(self.raw.as_ptr()) } != 0 { std::mem::forget(self.device.clone()); + std::mem::forget(self.memory.take()); self.quota.as_ref().unwrap().quarantine(); std::mem::forget(self.quota.take()); } @@ -945,6 +1011,7 @@ impl NativeQueuePair { post: impl FnOnce() -> c_int, ) -> Result<()> { if (region.is_implicit() && (recv || opcode != opcode::RDMA_WRITE)) + || (region.is_device() && !matches!(opcode, opcode::BIND_MW | opcode::LOCAL_INV)) || !Rc::ptr_eq(®ion.device, &self.device) || !region.contains(offset, length) || self.pending.borrow().contains_key(&id) @@ -1195,6 +1262,20 @@ mod tests { assert!(api.library.is_some(), "must load C adapter, not simulation"); } + #[test] + /// The C adapter reports the ABI version Rust requires and exports the + /// dma-buf registration with the signature Rust calls. + fn c_adapter_abi_version_and_dmabuf_signature_match() { + let source = include_str!("../native/verbs.c"); + assert!(source.contains(&format!( + "uint32_t rdma_verbs_abi(void) {{ return {ABI_VERSION}; }}" + ))); + assert!(source.contains( + "void *rdma_verbs_register_dmabuf(struct rdma_verbs_device *d, uint64_t offset, \ + uint64_t length,\n uint64_t iova, int fd)" + )); + } + #[test] /// Reject queue pair numbers outside the native 24-bit field. fn endpoint_rejects_invalid_native_parameters() { @@ -1290,6 +1371,17 @@ pub(crate) mod lifetime_tests { std::ptr::null_mut() } + /// The fixture device has no dma-buf support. + unsafe extern "C" fn register_dmabuf( + _: *mut c_void, + _: u64, + _: u64, + _: u64, + _: c_int, + ) -> *mut c_void { + std::ptr::null_mut() + } + /// Allocate a fixture queue pair with a fixed valid number. unsafe extern "C" fn qp(_: *mut c_void, _: u8, _: u32, _: u32, qpn: *mut u32) -> *mut c_void { unsafe { @@ -1452,6 +1544,7 @@ pub(crate) mod lifetime_tests { qp_free, register, register_implicit, + register_dmabuf, deregister, bytes, window, diff --git a/cmd/racer-dataplane/verbs/src/lib.rs b/cmd/racer-dataplane/verbs/src/lib.rs index 4ccc6e4b9..a0e6ba12a 100644 --- a/cmd/racer-dataplane/verbs/src/lib.rs +++ b/cmd/racer-dataplane/verbs/src/lib.rs @@ -77,7 +77,7 @@ pub use node::{ BACKOFF_MAX, BACKOFF_MIN, BACKOFF_RESET, COMMAND_QUEUE, Client, LinkKey, LinkSelector, LinkState, NodeConfig, Opened, Owner, OwnerPort, SETUP_TIMEOUT, Setup, node, }; -pub use pool::{BufferState, RegisteredPage}; +pub use pool::{BufferState, DevicePage, RegisteredPage}; #[cfg(any(test, feature = "simulation"))] pub use ffi::simulation; @@ -100,6 +100,37 @@ pub fn inventory() -> Result> { .collect()) } +/// Memory outside this crate that the NIC reaches through a Linux dma-buf, +/// such as GPU memory exported with `cuMemGetHandleForAddressRange`. +/// +/// A device pool built over it (see [`Client::attach_device`]) lets peers +/// write pages straight into this memory. The CPU never maps or copies it: +/// device pages can only receive, and their bytes are only reachable through +/// the owner (for example a GPU kernel). +/// +/// The crate keeps the owning [`DeviceMemory`] alive while the memory is +/// registered. If deregistration fails, the owner is leaked on purpose, like a +/// [`Guard`], so the memory is never freed while the NIC may still write it. +pub trait DmabufMemory: Send + Sync { + /// The dma-buf descriptor. Read only on the native thread, during + /// registration. The kernel keeps its own reference after that. + fn fd(&self) -> std::os::fd::BorrowedFd<'_>; + + /// Byte offset of this memory inside the dma-buf. + fn offset(&self) -> u64; + + /// Usable bytes from [`Self::offset`]. Split into page buffers of the + /// node's buffer length. + fn length(&self) -> usize; + + /// The address the NIC uses for the first byte. Windows over this memory + /// bind from it. + fn iova(&self) -> u64; +} + +/// The owner of a device pool's memory. See [`DmabufMemory`]. +pub type DeviceMemory = Arc; + /// Any value you want kept alive as long as a slot's RDMA resources live. /// /// Use it to count admission. If a resource is leaked, its guard leaks too. diff --git a/cmd/racer-dataplane/verbs/src/link.rs b/cmd/racer-dataplane/verbs/src/link.rs index c6dcb0bd9..ccc7d8be8 100644 --- a/cmd/racer-dataplane/verbs/src/link.rs +++ b/cmd/racer-dataplane/verbs/src/link.rs @@ -22,7 +22,7 @@ use crate::{ ffi::{ Access, MemoryWindow, NativeDevice, NativeQueuePair, NativeRegion, WorkCompletion, opcode, }, - pool::{Buffer, BufferState, Pool, RegisteredPage}, + pool::{Buffer, BufferState, DevicePage, Pool, RegisteredPage}, }; /// A link not `Ready` this long after creation breaks. @@ -63,6 +63,10 @@ pub enum Arrival { /// CPU access. Page(RegisteredPage), + /// Like [`Self::Page`], for a grant made with [`crate::Client::grant_device`]: + /// the bytes are in device memory and the CPU never touched them. + Device(DevicePage), + /// The peer sent `Release`. The buffer is already back in the pool. Released, } @@ -383,6 +387,9 @@ pub(crate) struct Link { windows: Vec>, + /// The device, for windows over a device pool, made on first use. + device: Rc, + control: Rc, epoch: Epoch, @@ -414,6 +421,10 @@ pub(crate) struct Link { free_windows: Vec, + /// Free windows tied to the device pool. A window stays tied to the + /// first region it binds, so device grants never reuse host windows. + free_device_windows: Vec, + free_slots: Vec, grants: BTreeMap, @@ -491,6 +502,8 @@ impl Link { qp, free_windows: (0..windows.len()).rev().collect(), windows, + device: device.clone(), + free_device_windows: Vec::new(), control, epoch, role, @@ -603,10 +616,15 @@ impl Link { } /// Bind a window over a free buffer and queue a `Grant`. `done` gets the - /// tag and arrival op once the `Grant` is posted. + /// tag and arrival op once the `Grant` is posted. With `device`, the + /// buffer comes from the device pool and the arrival is + /// [`Arrival::Device`]: fails with [`Error::Unavailable`] if the rail has + /// no device pool and [`Error::Overloaded`] if none is free. + #[allow(clippy::too_many_arguments)] pub(crate) fn grant( &mut self, pool: &Pool, + device: bool, transfer: [u8; 16], length: u32, deadline: Instant, @@ -628,12 +646,42 @@ impl Link { return; } - let Some(buffer) = pool.take(BufferState::Binding) else { + if device && pool.device_shared().is_none() { + let _ = done.complete(Err(Error::Unavailable)); + return; + } + + let window = if device { + match self.free_device_windows.pop() { + Some(window) => window, + None => match MemoryWindow::new(&self.device) { + // At most one device window per concurrent grant, so + // device windows never exceed the grant cap. + Ok(window) => { + self.windows.push(window); + self.windows.len() - 1 + } + Err(error) => { + let _ = done.complete(Err(error)); + return; + } + }, + } + } else { + self.free_windows.pop().expect("a window per grant") + }; + + let taken = if device { + pool.take_device(BufferState::Binding) + } else { + pool.take(BufferState::Binding) + }; + let Some(buffer) = taken else { + self.free_window(device, window); let _ = done.complete(Err(Error::Overloaded)); return; }; - let window = self.free_windows.pop().expect("a window per grant"); let tag = self.allocate_tag(); self.last_used = now; self.grants.insert( @@ -653,6 +701,15 @@ impl Link { self.outbox.push_back(Outgoing::Bind(tag)); } + /// Return `window` to its free list. + fn free_window(&mut self, device: bool, window: usize) { + if device { + self.free_device_windows.push(window); + } else { + self.free_windows.push(window); + } + } + /// Next nonzero tag not in use. fn allocate_tag(&mut self) -> u32 { loop { @@ -880,22 +937,27 @@ impl Link { } InFlight::Invalidate(tag) => { let mut entry = self.grants.remove(&tag).ok_or(Break::Error)?; - self.free_windows.push(entry.window); let buffer = entry.buffer.take().ok_or(Break::Error)?; + self.free_window(buffer.is_device(), entry.window); let GrantState::Invalidating(outcome) = entry.state else { return Err(Break::Error); }; match outcome { Outcome::Written(len) => { - let page = pool.own(buffer, len as usize).map_err(|buffer| { + let page = if buffer.is_device() { + pool.own_device(buffer, len as usize).map(Arrival::Device) + } else { + pool.own(buffer, len as usize).map(Arrival::Page) + } + .map_err(|buffer| { self.quarantine.push(buffer); Break::Error })?; if let Some(arrival) = entry.arrival.take() { // A dropped op hands the page back; dropping it frees the buffer. - let _ = arrival.complete(Ok(Arrival::Page(page))); + let _ = arrival.complete(Ok(page)); } } Outcome::Released => { @@ -1172,7 +1234,7 @@ impl Link { let key = self.qp.bind_window( wr(BIND, tag), &self.windows[entry.window], - pool.region(), + pool.region_of(buffer), pool.offset(buffer), entry.length, )?; @@ -1229,7 +1291,7 @@ impl Link { let message = ControlKind::Grant { tag, rkey: entry.key, - addr: pool.region().address() + pool.offset(buffer), + addr: pool.region_of(buffer).address() + pool.offset(buffer), length: entry.length, transfer_id: entry.transfer, }; diff --git a/cmd/racer-dataplane/verbs/src/link/tests.rs b/cmd/racer-dataplane/verbs/src/link/tests.rs index 4c1125347..8bed8cb41 100644 --- a/cmd/racer-dataplane/verbs/src/link/tests.rs +++ b/cmd/racer-dataplane/verbs/src/link/tests.rs @@ -142,7 +142,7 @@ fn poll(op: &Op) -> Option> { fn grant(side: &mut Side, length: u32, deadline: Instant, now: Instant) -> Op { let (granted, done) = op(); side.link - .grant(&side.pool, [7; 16], length, deadline, done, now); + .grant(&side.pool, false, [7; 16], length, deadline, done, now); granted } @@ -182,7 +182,7 @@ fn transfer(p: &mut Pair, to_a: bool, bytes: &[u8]) -> RegisteredPage { assert_eq!(poll(&written), Some(Ok(()))); match poll(&arrival).unwrap().unwrap() { Arrival::Page(page) => page, - Arrival::Released => panic!("expected a page"), + Arrival::Released | Arrival::Device(_) => panic!("expected a page"), } } @@ -1017,7 +1017,7 @@ fn direct_write_reads_in_place_and_holds_the_source_until_completion() { assert_eq!(Arc::strong_count(&source), 1, "released on completion"); match poll(&arrival).unwrap().unwrap() { Arrival::Page(page) => assert_eq!(page.bytes(), &source[..]), - Arrival::Released => panic!("expected a page"), + Arrival::Released | Arrival::Device(_) => panic!("expected a page"), } finish(p); } @@ -1047,7 +1047,7 @@ fn direct_writes_accept_boxed_and_static_sources() { assert_eq!(Arc::strong_count(&source), 1, "released on completion"); match poll(&arrival).unwrap().unwrap() { Arrival::Page(page) => assert_eq!(page.bytes(), &expected[..]), - Arrival::Released => panic!("expected a page"), + Arrival::Released | Arrival::Device(_) => panic!("expected a page"), } } finish(p); diff --git a/cmd/racer-dataplane/verbs/src/node.rs b/cmd/racer-dataplane/verbs/src/node.rs index 2a3a38456..23dac0c2b 100644 --- a/cmd/racer-dataplane/verbs/src/node.rs +++ b/cmd/racer-dataplane/verbs/src/node.rs @@ -23,7 +23,7 @@ use std::{ use futures::task::AtomicWaker; use crate::{ - Endpoint, Error, Guard, GuardOwner, PortInfo, Result, Selection, + DeviceMemory, Endpoint, Error, Guard, GuardOwner, PortInfo, Result, Selection, channel::{self, Completer, Op, op}, ffi::{self, Access, NativeDevice, NativeRegion}, link::{self, Break, Epoch, Granted, Limits, Link, Phase, Role, Source}, @@ -157,9 +157,16 @@ enum Command { done: Completer<()>, }, + AttachDevice { + rail: u16, + memory: DeviceMemory, + done: Completer, + }, + Grant { key: LinkKey, epoch: Epoch, + device: bool, transfer: [u8; 16], length: u32, deadline: Instant, @@ -191,6 +198,9 @@ struct NodeShared { pools: RwLock>>, + /// Device pools by rail, once attached. + device_pools: RwLock>>, + /// Breaks that did not fit in a command queue. breaks: Mutex>, @@ -214,6 +224,7 @@ pub fn node(workers: usize) -> Result<(Vec, OwnerPort)> { let shared = Arc::new(NodeShared { links: RwLock::new(BTreeMap::new()), pools: RwLock::new(BTreeMap::new()), + device_pools: RwLock::new(BTreeMap::new()), breaks: Mutex::new(Vec::new()), closed: AtomicBool::new(false), drained: AtomicBool::new(false), @@ -344,6 +355,7 @@ impl Client { self.submit(|done| Command::Grant { key, epoch, + device: false, transfer, length, deadline, @@ -351,6 +363,41 @@ impl Client { }) } + /// Receiver: like [`Self::grant`], but over a free buffer of the rail's + /// device pool (see [`Self::attach_device`]). The arrival is + /// [`crate::Arrival::Device`]. The fence is the same: the page is handed + /// out only after the peer's write completion and the window's signaled + /// invalidation, so the peer can no longer write it. Fails with + /// `Unavailable` if the rail has no device pool, and `Overloaded` if no + /// device buffer is free. Nothing falls back to host memory. + pub fn grant_device( + &self, + key: LinkKey, + epoch: Epoch, + transfer: [u8; 16], + length: u32, + deadline: Instant, + ) -> Op { + self.submit(|done| Command::Grant { + key, + epoch, + device: true, + transfer, + length, + deadline, + done, + }) + } + + /// Register `memory` (a dma-buf, such as GPU memory) on `rail` as its + /// device pool, split into page buffers of the node's buffer length. + /// Once per rail per round, after [`Self::activate`]. Resolves to the + /// number of device buffers. The memory stays registered until the node + /// drains; it is never read or written by the CPU. + pub fn attach_device(&self, rail: u16, memory: DeviceMemory) -> Op { + self.submit(|done| Command::AttachDevice { rail, memory, done }) + } + /// Sender: write `source` into the peer's grant `tag` with immediate, /// waiting for its `Grant` until `deadline`. Resolves on the send /// completion. Dropping the op does not cancel a posted write. @@ -405,6 +452,13 @@ impl Client { Some(pools.get(&rail)?.count_in(state)) } + /// Device page buffers of `rail` in `state`, for diagnostics. `None` + /// without a device pool. + pub fn device_buffers(&self, rail: u16, state: BufferState) -> Option { + let pools = self.shared.device_pools.try_read().ok()?; + Some(pools.get(&rail)?.count_in(state)) + } + /// Close the node: the owner sends `Close` on ready links, breaks every /// link, and frees everything once no page is held. Does not wait. pub fn close(&self) { @@ -554,7 +608,7 @@ impl Owner { self.shared.owner.register(waker); for device in self.devices.values() { - device.pool.shared().register_owner(waker); + device.pool.register_owner(waker); } } @@ -661,15 +715,21 @@ impl Owner { } => { let _ = done.complete(self.connect(key, epoch, remote, link_id, now)); } + Command::AttachDevice { rail, memory, done } => { + let _ = done.complete(self.attach_device(rail, memory)); + } Command::Grant { key, epoch, + device, transfer, length, deadline, done, } => match self.ready(key, epoch) { - Some((link, pool)) => link.grant(pool, transfer, length, deadline, done, now), + Some((link, pool)) => { + link.grant(pool, device, transfer, length, deadline, done, now) + } None => { let _ = done.complete(Err(Error::Unavailable)); } @@ -729,6 +789,33 @@ impl Owner { } } + /// Register `memory` as `rail`'s device pool. + fn attach_device(&mut self, rail: u16, memory: DeviceMemory) -> Result { + #[cfg(any(test, feature = "simulation"))] + let _simulation = self + .simulation + .as_ref() + .map(crate::simulation::Simulation::enter); + let quota = self.quota.clone().ok_or(Error::InvalidRequest)?; + let state = self.devices.get_mut(&rail).ok_or(Error::InvalidRequest)?; + + if state.pool.device_shared().is_some() { + return Err(Error::InvalidRequest); + } + + let region = NativeRegion::dmabuf(state.device.clone(), memory, quota)?; + state.pool.attach_device(region)?; + // The driver waker reaches the device pool on its next registration. + let shared = state.pool.device_shared().expect("just attached").clone(); + let count = shared.count(); + self.shared + .device_pools + .write() + .unwrap_or_else(PoisonError::into_inner) + .insert(rail, shared); + Ok(count) + } + fn activate(&mut self, config: NodeConfig) -> Result> { if self.settings.is_some() || !self.devices.is_empty() { return Err(Error::InvalidRequest); @@ -1052,6 +1139,11 @@ impl Owner { Err(_) => return, } + match self.shared.device_pools.try_write() { + Ok(mut pools) => pools.clear(), + Err(_) => return, + } + if self.devices.values().any(|d| !d.pool.idle()) { return; } @@ -1155,6 +1247,11 @@ impl Drop for Owner { .write() .unwrap_or_else(PoisonError::into_inner) .clear(); + self.shared + .device_pools + .write() + .unwrap_or_else(PoisonError::into_inner) + .clear(); // Pools leak their regions if pages are still held (see `pool.rs`). self.devices.clear(); } @@ -1169,6 +1266,7 @@ fn fail(command: Command, error: Error) { | Command::Write { done, .. } | Command::Release { done, .. } => drop(done.complete(Err(error))), Command::Grant { done, .. } => drop(done.complete(Err(error))), + Command::AttachDevice { done, .. } => drop(done.complete(Err(error))), Command::Break(..) => {} } } diff --git a/cmd/racer-dataplane/verbs/src/pool.rs b/cmd/racer-dataplane/verbs/src/pool.rs index 56c64aed2..19127544e 100644 --- a/cmd/racer-dataplane/verbs/src/pool.rs +++ b/cmd/racer-dataplane/verbs/src/pool.rs @@ -14,6 +14,12 @@ //! `Owned` only after the fence (receive completion, then a signaled //! `LOCAL_INV` completion). //! +//! A rail may also hold a device pool: page buffers in caller memory reached +//! through a dma-buf (for example GPU memory). Its buffers use the same +//! states and the same fence, but a fenced device buffer becomes a +//! [`DevicePage`], which has no slice or pointer API: the CPU never reads or +//! copies device memory. +//! //! Page buffers only receive. Writes read caller memory in place through the //! device's implicit on-demand paging region, which every pool holds. //! @@ -35,7 +41,7 @@ use std::{ use futures::task::AtomicWaker; -use crate::{Error, GuardOwner, Result, ffi::NativeRegion}; +use crate::{DeviceMemory, Error, GuardOwner, Result, ffi::NativeRegion}; /// Ownership state of one page buffer. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -98,9 +104,12 @@ fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { /// State shared by the owner and every received page. pub(crate) struct PoolShared { /// Start of the registered memory. Valid while any `Arc` exists (see the - /// module docs). + /// module docs). Null for a device pool. base: *mut u8, + /// The dma-buf owner of a device pool. + memory: Option, + buffer_len: usize, states: Box<[AtomicU8]>, @@ -174,6 +183,24 @@ impl PoolShared { #[derive(Debug, Eq, PartialEq)] pub(crate) struct Buffer { index: u32, + + /// In the device pool. + device: bool, +} + +impl Buffer { + /// True for a device pool buffer. + pub(crate) fn is_device(&self) -> bool { + self.device + } +} + +/// A rail's device pool. See the module docs. +struct DevicePool { + shared: Arc, + + /// `None` only during drop. + region: Option>, } /// The owner's side of one device's page buffers. Not `Send`. @@ -187,6 +214,9 @@ pub(crate) struct Pool { /// every write source. `None` only during drop. implicit: Option>, + /// Device page buffers, once attached. + device: Option, + quota: Arc, } @@ -220,6 +250,7 @@ impl Pool { Ok(Self { shared: Arc::new(PoolShared { base, + memory: None, buffer_len, states: (0..count) .map(|_| AtomicU8::new(BufferState::Free.code())) @@ -230,10 +261,112 @@ impl Pool { }), region: Some(region), implicit: Some(implicit), + device: None, quota, }) } + /// Split the dma-buf `region` (from [`NativeRegion::dmabuf`]) into + /// device page buffers of the host buffer length. At most one per pool. + pub(crate) fn attach_device(&mut self, region: Rc) -> Result<()> { + if self.device.is_some() { + return Err(Error::InvalidRequest); + } + + let Some(memory) = region.memory().cloned() else { + return Err(Error::InvalidConfiguration); + }; + + let buffer_len = self.shared.buffer_len; + let count = region.length() / buffer_len; + if count == 0 || count > u32::MAX as usize { + return Err(Error::InvalidConfiguration); + } + + self.device = Some(DevicePool { + shared: Arc::new(PoolShared { + base: std::ptr::null_mut(), + memory: Some(memory), + buffer_len, + states: (0..count) + .map(|_| AtomicU8::new(BufferState::Free.code())) + .collect(), + free: Mutex::new((0..count as u32).rev().collect()), + owner: AtomicWaker::new(), + }), + region: Some(region), + }); + Ok(()) + } + + /// State shared with device pages, if a device pool is attached. + pub(crate) fn device_shared(&self) -> Option<&Arc> { + self.device.as_ref().map(|d| &d.shared) + } + + /// Register the owner's waker for buffers returned to either pool. + pub(crate) fn register_owner(&self, waker: &Waker) { + self.shared.register_owner(waker); + if let Some(device) = &self.device { + device.shared.register_owner(waker); + } + } + + /// State shared by `buffer`'s pool. + fn shared_of(&self, buffer: &Buffer) -> &Arc { + if buffer.device { + &self + .device + .as_ref() + .expect("device buffer without pool") + .shared + } else { + &self.shared + } + } + + /// The registered region `buffer` lives in. + pub(crate) fn region_of(&self, buffer: &Buffer) -> &Rc { + if buffer.device { + let device = self.device.as_ref().expect("device buffer without pool"); + device.region.as_ref().unwrap() + } else { + self.region() + } + } + + /// Take a free device buffer straight into an owner `state`. `None` if + /// no device pool is attached or none is free. + pub(crate) fn take_device(&self, state: BufferState) -> Option { + assert!(state.owner_held() && state != BufferState::Quarantined); + let device = self.device.as_ref()?; + device.shared.pop(state).map(|index| Buffer { + index, + device: true, + }) + } + + /// Fence done after a write into a device buffer: hand `len` bytes to + /// the caller as a [`DevicePage`]. + pub(crate) fn own_device( + &self, + buffer: Buffer, + len: usize, + ) -> std::result::Result { + if !buffer.device || len == 0 || len > self.shared.buffer_len { + return Err(buffer); + } + + let shared = self.shared_of(&buffer).clone(); + shared.set(buffer.index, BufferState::Owned); + + Ok(DevicePage { + shared, + index: buffer.index, + len, + }) + } + /// State shared with received pages. pub(crate) fn shared(&self) -> &Arc { &self.shared @@ -257,18 +390,21 @@ impl Pool { /// Take a free buffer straight into an owner `state`. pub(crate) fn take(&self, state: BufferState) -> Option { assert!(state.owner_held() && state != BufferState::Quarantined); - self.shared.pop(state).map(|index| Buffer { index }) + self.shared.pop(state).map(|index| Buffer { + index, + device: false, + }) } /// Move an owner-held buffer to another owner `state`. pub(crate) fn set(&self, buffer: &Buffer, state: BufferState) { assert!(state.owner_held()); - self.shared.set(buffer.index, state); + self.shared_of(buffer).set(buffer.index, state); } /// Fence done (or nothing was ever exposed): return to `Free`. pub(crate) fn release(&self, buffer: Buffer) { - self.shared.push(buffer.index); + self.shared_of(&buffer).push(buffer.index); } /// Fence done after a write: hand `len` bytes to the CPU as `Owned`. @@ -277,7 +413,7 @@ impl Pool { buffer: Buffer, len: usize, ) -> std::result::Result { - if len == 0 || len > self.shared.buffer_len { + if buffer.device || len == 0 || len > self.shared.buffer_len { return Err(buffer); } @@ -292,18 +428,26 @@ impl Pool { /// Destroy failed: the buffer is never reused. pub(crate) fn leak(&self, buffer: Buffer) { - self.shared.set(buffer.index, BufferState::Leaked); + self.shared_of(&buffer) + .set(buffer.index, BufferState::Leaked); } /// Received pages still alive outside the owner. pub(crate) fn outstanding(&self) -> usize { Arc::strong_count(&self.shared) - 1 + + self + .device + .as_ref() + .map_or(0, |d| Arc::strong_count(&d.shared) - 1) } /// True if dropping now would deregister the region: no buffer is /// outside `Free`. pub(crate) fn idle(&self) -> bool { - self.outstanding() == 0 && self.shared.count_in(BufferState::Free) == self.shared.count() + let free = |shared: &PoolShared| shared.count_in(BufferState::Free) == shared.count(); + self.outstanding() == 0 + && free(&self.shared) + && self.device.as_ref().is_none_or(|d| free(&d.shared)) } } @@ -316,6 +460,9 @@ impl Drop for Pool { std::mem::forget(self.quota.clone()); std::mem::forget(self.region.take()); std::mem::forget(self.implicit.take()); + if let Some(device) = &mut self.device { + std::mem::forget(device.region.take()); + } } } } @@ -366,6 +513,59 @@ impl Drop for RegisteredPage { } } +/// A received device page: no window covers it and no work request uses it. +/// +/// The bytes are `memory()[offset()..offset() + len()]`, in memory the CPU +/// never maps. Read them with the owner of the memory, such as a GPU kernel, +/// and keep this value alive until that work is done. `Send` and `Sync`. +/// Dropping it, on any thread, returns the buffer to the device pool, after +/// which the next peer write may land there. +pub struct DevicePage { + shared: Arc, + + index: u32, + + len: usize, +} + +impl DevicePage { + /// The memory the peer wrote into. + pub fn memory(&self) -> &DeviceMemory { + self.shared.memory.as_ref().expect("device pool memory") + } + + /// Offset of the first received byte from the start of + /// [`Self::memory`] (after [`crate::DmabufMemory::offset`]). + pub fn offset(&self) -> u64 { + self.index as u64 * self.shared.buffer_len as u64 + } + + /// Number of received bytes. + pub fn len(&self) -> usize { + self.len + } + + /// True if no bytes were received. Never true for a delivered page. + pub fn is_empty(&self) -> bool { + self.len == 0 + } +} + +impl std::fmt::Debug for DevicePage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DevicePage") + .field("index", &self.index) + .field("len", &self.len) + .finish() + } +} + +impl Drop for DevicePage { + fn drop(&mut self) { + self.shared.push(self.index); + } +} + #[cfg(test)] mod tests { use std::{ @@ -542,4 +742,61 @@ mod tests { assert_eq!(sim.live_resources(), 3); } } + + #[test] + /// A device pool shares the buffer states and fence, hands out + /// `DevicePage` instead of bytes, keeps host and device buffers apart, + /// and leaks its region with the host region when not idle. + fn device_pool_buffers_and_pages() { + let sim = Simulation::new(); + let device = self::device(&sim); + let (quota, charge) = lifetime_tests::quota(); + let region = + NativeRegion::with_access(device.clone(), LEN, Access::PAGES, quota.clone()).unwrap(); + let implicit = NativeRegion::implicit(device.clone(), quota.clone()).unwrap(); + let mut pool = Pool::new(region, implicit, LEN, quota).unwrap(); + let memory = crate::simulation::MemfdMemory::new(2 * LEN + 1, 0x9000).unwrap(); + let (quota, _observer) = lifetime_tests::quota(); + assert!(pool.take_device(BufferState::Binding).is_none()); + // Host regions are refused as device pools. + let host = + NativeRegion::with_access(device.clone(), LEN, Access::PAGES, quota.clone()).unwrap(); + assert_eq!(pool.attach_device(host), Err(Error::InvalidConfiguration)); + let region = NativeRegion::dmabuf(device.clone(), memory.clone(), quota.clone()).unwrap(); + pool.attach_device(region.clone()).unwrap(); + assert_eq!(pool.attach_device(region), Err(Error::InvalidRequest)); + let shared = pool.device_shared().unwrap().clone(); + assert_eq!(shared.count(), 2); + let buffer = pool.take_device(BufferState::Binding).unwrap(); + assert!(buffer.is_device()); + assert!(pool.region_of(&buffer).is_device()); + assert_eq!(pool.region_of(&buffer).address(), 0x9000); + pool.set(&buffer, BufferState::Invalidating); + assert_eq!(shared.state(0), BufferState::Invalidating); + assert_eq!(pool.shared().state(0), BufferState::Free, "host untouched"); + // A device buffer never becomes a host page with bytes. + let buffer = pool.own(buffer, 10).unwrap_err(); + let page = pool.own_device(buffer, 10).unwrap(); + assert_eq!((page.offset(), page.len()), (0, 10)); + assert_eq!(shared.state(0), BufferState::Owned); + let second = pool.take_device(BufferState::Binding).unwrap(); + let second = pool.own_device(second, LEN).unwrap(); + assert_eq!(second.offset(), LEN as u64); + assert!(pool.take_device(BufferState::Binding).is_none()); + let host = pool.take(BufferState::Binding).unwrap(); + let host = pool.own_device(host, 1).unwrap_err(); + pool.release(host); + drop(second); + assert!(!pool.idle(), "a device page is outstanding"); + drop(shared); + drop(pool); + // Dropped while a device page lives: both regions leak with the charge. + assert_eq!(page.memory().length(), 2 * LEN + 1); + drop(page); + assert_eq!(charge.get(), 1); + assert!( + Arc::strong_count(&memory) > 1, + "leaked region keeps the owner" + ); + } } diff --git a/cmd/racer-dataplane/verbs/src/simulation.rs b/cmd/racer-dataplane/verbs/src/simulation.rs index e15d0a120..809303eca 100644 --- a/cmd/racer-dataplane/verbs/src/simulation.rs +++ b/cmd/racer-dataplane/verbs/src/simulation.rs @@ -16,6 +16,11 @@ //! 5. Check what happened with [`Simulation::trace`]. //! //! Addresses seen on the wire are fake; they are never host pointers. +//! A dma-buf region maps the given descriptor, so a memfd from +//! [`MemfdMemory`] stands in for GPU memory. Its wire address is the iova. +//! Like the C adapter, sends, writes, and receives that name a dma-buf region +//! are rejected, and [`Operation::RegisterDmabuf`] traces and faults its +//! registration. //! Device names must be unique within a discovery view, even for different ports. //! A remote QP must already exist in this fabric before its peer connects to it. //! @@ -110,6 +115,8 @@ pub enum Operation { Register, + RegisterDmabuf, + Deregister, Window, @@ -225,7 +232,7 @@ enum Resource { Region { device: u32, - bytes: Box<[u8]>, + bytes: Memory, address: u64, @@ -297,6 +304,10 @@ const MR_CONTROL: u32 = 2; /// `RDMA_VERBS_MR_REMOTE_WRITE`. const MR_REMOTE_WRITE: u32 = 4; +/// A dma-buf region from `rdma_verbs_register_dmabuf`. Peers write through +/// windows; never a send, write, or receive buffer. +const MR_DEVICE: u32 = 16; + /// An implicit on-demand paging region over this process's address space. /// Offsets are virtual addresses; it is only a local write or send source. const MR_IMPLICIT: u32 = 8; @@ -538,6 +549,7 @@ pub(super) fn api() -> Rc { qp_free, register, register_implicit, + register_dmabuf, deregister, bytes, window, @@ -1370,7 +1382,7 @@ unsafe extern "C" fn register(device: *mut c_void, length: u64, access: u32) -> &h.sim, Resource::Region { device: h.id, - bytes: vec![0; length as usize].into_boxed_slice(), + bytes: Memory::Host(vec![0; length as usize].into_boxed_slice()), address, access, }, @@ -1401,7 +1413,7 @@ unsafe extern "C" fn register_implicit(device: *mut c_void) -> *mut c_void { &h.sim, Resource::Region { device: h.id, - bytes: Box::default(), + bytes: Memory::Host(Box::default()), address: 0, access: MR_IMPLICIT, }, @@ -1413,13 +1425,213 @@ unsafe extern "C" fn register_implicit(device: *mut c_void) -> *mut c_void { raw } -/// Pointer to a region's bytes. Valid until the region is freed. +/// Map a dma-buf as a page region whose wire address is `iova`. Like C, +/// `fd` is not kept: the mapping holds its own reference. +unsafe extern "C" fn register_dmabuf( + device: *mut c_void, + offset: u64, + length: u64, + iova: u64, + fd: c_int, +) -> *mut c_void { + let Some(h) = (unsafe { handle(device) }) else { + return std::ptr::null_mut(); + }; + let mut world = h.sim.world.borrow_mut(); + if world.rejected(Operation::RegisterDmabuf, h.id) { + return std::ptr::null_mut(); + } + let mapped = (length != 0 && fd >= 0 && usize::try_from(length).is_ok()) + .then(|| map(fd, offset, length as usize)) + .flatten(); + let Some(pointer) = mapped else { + world.record(Operation::RegisterDmabuf, h.id, None, -1, false); + return std::ptr::null_mut(); + }; + drop(world); + let raw = allocate( + &h.sim, + Resource::Region { + device: h.id, + bytes: Memory::Mapped { + pointer, + length: length as usize, + }, + address: iova, + access: MR_PAGES | MR_DEVICE, + }, + ); + h.sim + .world + .borrow_mut() + .record(Operation::RegisterDmabuf, h.id, None, 0, false); + raw +} + +/// A region's bytes: owned host memory, or a shared mapping of a dma-buf. +enum Memory { + Host(Box<[u8]>), + + Mapped { pointer: NonNull, length: usize }, +} + +impl std::ops::Deref for Memory { + type Target = [u8]; + + /// The region's bytes. + fn deref(&self) -> &[u8] { + match self { + Self::Host(bytes) => bytes, + // SAFETY: the mapping lives until this value drops. + Self::Mapped { pointer, length } => unsafe { + std::slice::from_raw_parts(pointer.as_ptr(), *length) + }, + } + } +} + +impl std::ops::DerefMut for Memory { + /// The region's bytes, writable. + fn deref_mut(&mut self) -> &mut [u8] { + match self { + Self::Host(bytes) => bytes, + // SAFETY: the mapping lives until this value drops. + Self::Mapped { pointer, length } => unsafe { + std::slice::from_raw_parts_mut(pointer.as_ptr(), *length) + }, + } + } +} + +impl Drop for Memory { + /// Unmap a dma-buf mapping. The owner's memory stays. + fn drop(&mut self) { + if let Self::Mapped { pointer, length } = self { + unsafe { + libc::munmap(pointer.as_ptr().cast(), *length); + } + } + } +} + +/// Map `length` bytes of `fd` from `offset`, shared and writable. +fn map(fd: c_int, offset: u64, length: usize) -> Option> { + let offset = libc::off_t::try_from(offset).ok()?; + let pointer = unsafe { + libc::mmap( + std::ptr::null_mut(), + length, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + fd, + offset, + ) + }; + if pointer == libc::MAP_FAILED { + return None; + } + NonNull::new(pointer.cast()) +} + +/// Host memory standing in for a GPU dma-buf: a memfd the simulation maps. +/// +/// The simulated NIC writes through its own shared mapping, so +/// [`Self::read`] sees exactly what a peer wrote. +pub struct MemfdMemory { + fd: std::os::fd::OwnedFd, + + view: NonNull, + + length: usize, + + iova: u64, +} + +// SAFETY: the mapping is plain shared memory, valid for this value's lifetime. +unsafe impl Send for MemfdMemory {} + +unsafe impl Sync for MemfdMemory {} + +impl MemfdMemory { + /// A zeroed memfd of `length` bytes that windows address from `iova`. + pub fn new(length: usize, iova: u64) -> std::io::Result> { + use std::os::fd::{AsRawFd, FromRawFd}; + + if length == 0 { + return Err(std::io::ErrorKind::InvalidInput.into()); + } + let raw = unsafe { libc::memfd_create(c"rdma-verbs-sim".as_ptr(), libc::MFD_CLOEXEC) }; + if raw < 0 { + return Err(std::io::Error::last_os_error()); + } + let fd = unsafe { std::os::fd::OwnedFd::from_raw_fd(raw) }; + if unsafe { libc::ftruncate(fd.as_raw_fd(), length as libc::off_t) } != 0 { + return Err(std::io::Error::last_os_error()); + } + let view = map(fd.as_raw_fd(), 0, length).ok_or_else(std::io::Error::last_os_error)?; + Ok(Arc::new(Self { + fd, + view, + length, + iova, + })) + } + + /// Copy `length` bytes at `offset` out, as a GPU kernel would read them. + pub fn read(&self, offset: usize, length: usize) -> Vec { + assert!( + offset + .checked_add(length) + .is_some_and(|end| end <= self.length) + ); + unsafe { std::slice::from_raw_parts(self.view.as_ptr().add(offset), length) }.to_vec() + } +} + +impl Drop for MemfdMemory { + /// Unmap this value's view. The memfd closes with `fd`. + fn drop(&mut self) { + unsafe { + libc::munmap(self.view.as_ptr().cast(), self.length); + } + } +} + +impl crate::DmabufMemory for MemfdMemory { + /// The memfd. + fn fd(&self) -> std::os::fd::BorrowedFd<'_> { + use std::os::fd::AsFd; + + self.fd.as_fd() + } + + /// Always 0. + fn offset(&self) -> u64 { + 0 + } + + /// The memfd size. + fn length(&self) -> usize { + self.length + } + + /// The iova given to [`Self::new`]. + fn iova(&self) -> u64 { + self.iova + } +} + +/// Pointer to a host region's bytes. Valid until the region is freed. Null +/// for an implicit or dma-buf region, like C. unsafe extern "C" fn bytes(raw: *mut c_void) -> *mut u8 { let Some(h) = (unsafe { handle(raw) }) else { return std::ptr::null_mut(); }; match h.sim.world.borrow_mut().resources.get_mut(&h.id) { - Some(Resource::Region { bytes, .. }) => bytes.as_mut_ptr(), + Some(Resource::Region { + bytes: Memory::Host(bytes), + .. + }) => bytes.as_mut_ptr(), _ => std::ptr::null_mut(), } } @@ -1506,7 +1718,11 @@ unsafe fn qp_and_region<'a>( let (Some(q), Some(r)) = (unsafe { (handle(qp), handle(region)) }) else { return None; }; - if !Rc::ptr_eq(&q.sim.world, &r.sim.world) { + let device = matches!( + r.sim.world.borrow().resources.get(&r.id), + Some(Resource::Region { access, .. }) if access & MR_DEVICE != 0 + ); + if device || !Rc::ptr_eq(&q.sim.world, &r.sim.world) { q.sim .world .borrow_mut() @@ -2100,6 +2316,57 @@ mod tests { assert_eq!(foreign.live_resources(), 0); } + #[test] + /// A dma-buf region has no CPU mapping and is never a send, write, or + /// receive buffer, in Rust or at the adapter: only windows reach it. + fn dmabuf_regions_reject_cpu_and_copy_paths() { + let sim = Simulation::new(); + { + let link = link(&sim, |_| {}); + let memory = MemfdMemory::new(8192, 0x5000_0000).unwrap(); + let (quota, _) = lifetime_tests::quota(); + let region = + NativeRegion::dmabuf(link.pages.device.clone(), memory.clone(), quota).unwrap(); + assert!(region.is_device() && !region.is_implicit()); + assert!(region.as_ptr().is_null()); + assert_eq!(region.address(), 0x5000_0000); + assert_eq!(region.length(), 8192); + assert_eq!( + link.a.send(wr(1), ®ion, 0, 16), + Err(Error::InvalidRequest) + ); + assert_eq!( + link.a.post_recv(wr(2), ®ion, 0, 16), + Err(Error::InvalidRequest) + ); + assert_eq!( + link.a + .write_imm(wr(3), ®ion, 0, 16, link.pages.address(), 1, 0), + Err(Error::InvalidRequest) + ); + sim.take_trace(); + let q = link.a.raw.as_ptr(); + let qpn = link.a.endpoint.qpn; + unsafe { + assert!(bytes(region.raw.as_ptr()).is_null()); + assert_eq!(send(q, region.raw.as_ptr(), 0, 16, 51), -1); + events(&sim, &[(Operation::Send, qpn, Some(51), -1, false)]); + assert_eq!(post_recv(q, region.raw.as_ptr(), 0, 16, 52), -1); + events(&sim, &[(Operation::PostRecv, qpn, Some(52), -1, false)]); + let target = link.pages.address(); + assert_eq!( + write_imm(q, region.raw.as_ptr(), 0, 16, target, 1, 0, 53), + -1 + ); + events(&sim, &[(Operation::WriteImm, qpn, Some(53), -1, false)]); + } + drop(region); + assert_eq!(Arc::strong_count(&memory), 1, "deregister drops the owner"); + stop_both(&link); + } + assert_eq!(sim.live_resources(), 0); + } + /// Two connected QPs on one device with explicit depths, a page region, /// and a control region. struct Link { diff --git a/cmd/racer-dataplane/verbs/tests/links.rs b/cmd/racer-dataplane/verbs/tests/links.rs index db674dfa2..5da2706af 100644 --- a/cmd/racer-dataplane/verbs/tests/links.rs +++ b/cmd/racer-dataplane/verbs/tests/links.rs @@ -15,7 +15,7 @@ use std::{ use rdma_verbs::{ Arrival, BACKOFF_MIN, BufferState, COMMAND_QUEUE, Client, Epoch, Error, Granted, LinkKey, LinkState, NodeConfig, Op, Opened, Owner, SETUP_TIMEOUT, Setup, Source, - simulation::{Device, Fault, Operation, Simulation}, + simulation::{Device, Fault, MemfdMemory, Operation, Simulation}, }; use uring_runtime::environment::{Environment, SimulationClock}; @@ -197,7 +197,7 @@ fn bytes(data: Vec) -> Source { fn page(arrival: rdma_verbs::Result) -> rdma_verbs::RegisteredPage { match arrival.unwrap() { Arrival::Page(page) => page, - Arrival::Released => panic!("released"), + Arrival::Released | Arrival::Device(_) => panic!("expected a host page"), } } @@ -817,3 +817,169 @@ mod inner { } } } + +/// Device pool memory: two buffers plus a tail too small for a third. +fn device_memory() -> Arc { + MemfdMemory::new(2 * LEN + 5, 0x7000_0000).unwrap() +} + +fn device_page(arrival: rdma_verbs::Result) -> rdma_verbs::DevicePage { + match arrival.unwrap() { + Arrival::Device(page) => page, + other => panic!("expected a device page, got {other:?}"), + } +} + +#[test] +/// A device grant lands the peer's write in dma-buf memory with no CPU copy: +/// the bytes are only visible through the memory owner, the page holds its +/// buffer until dropped, and host grants keep using the host pool. +fn device_grants_land_in_dmabuf_memory() { + let mut w = world(); + let (ea, eb) = link_up(&mut w); + let memory = device_memory(); + let attached = w.a.clients[0].attach_device(7, memory.clone()); + assert_eq!(wait(&mut w, &attached), Ok(2)); + assert_eq!(w.a.clients[0].device_buffers(7, BufferState::Free), Some(2)); + assert_eq!(w.b.clients[0].device_buffers(7, BufferState::Free), None); + let deadline = now() + 5 * SECOND; + let mut held = Vec::new(); + for round in 0..2u8 { + let granted = w.a.clients[0].grant_device(TO_B, ea, [round; 16], 100, deadline); + let granted = wait(&mut w, &granted).unwrap(); + let data = vec![0xa0 | round; 100]; + let written = w.b.clients[1].write(TO_A, eb, granted.tag, bytes(data.clone()), deadline); + assert_eq!(wait(&mut w, &written), Ok(())); + let page = device_page(ready(&granted.arrival)); + assert_eq!((page.len(), page.is_empty()), (100, false)); + assert_eq!(page.offset(), u64::from(round) * LEN as u64); + assert!(Arc::ptr_eq( + page.memory(), + &(memory.clone() as rdma_verbs::DeviceMemory) + )); + assert_eq!(memory.read(page.offset() as usize, 100), data); + held.push(page); + } + assert_eq!( + w.a.clients[0].device_buffers(7, BufferState::Owned), + Some(2) + ); + // Both device buffers are held: the next device grant is refused, and + // nothing falls back to host memory. + let refused = w.a.clients[0].grant_device(TO_B, ea, [7; 16], 10, deadline); + assert_eq!(wait(&mut w, &refused).err(), Some(Error::Overloaded)); + assert_eq!(w.a.clients[0].buffers(7, BufferState::Free), Some(BUFFERS)); + // Host grants still use the host pool. + let host = w.a.clients[0].grant(TO_B, ea, [8; 16], 3, deadline); + let host = wait(&mut w, &host).unwrap(); + let written = w.b.clients[0].write(TO_A, eb, host.tag, bytes(vec![1, 2, 3]), deadline); + assert_eq!(wait(&mut w, &written), Ok(())); + assert_eq!(page(ready(&host.arrival)).bytes(), [1, 2, 3]); + // A page dropped on another thread frees its buffer for the next grant, + // and a release returns a device buffer without producing a page. + let page = held.pop().unwrap(); + std::thread::spawn(move || drop(page)).join().unwrap(); + pump(&mut w); + let granted = w.a.clients[0].grant_device(TO_B, ea, [9; 16], 10, deadline); + let granted = wait(&mut w, &granted).unwrap(); + let released = w.b.clients[0].release(TO_A, eb, granted.tag); + assert_eq!(wait(&mut w, &released), Ok(())); + assert!(matches!(ready(&granted.arrival), Ok(Arrival::Released))); + assert_eq!(w.a.clients[0].device_buffers(7, BufferState::Free), Some(1)); + // Shutdown waits for the held device page. + w.a.clients[0].close(); + pump(&mut w); + assert!(!w.a.clients[0].drained()); + drop(held); + pump(&mut w); + assert!(w.a.clients[0].drained()); + assert_eq!( + Arc::strong_count(&memory), + 1, + "the memory owner is released" + ); + shutdown(w); +} + +#[test] +/// Attach is once per activated rail; a device grant without a device pool +/// is unavailable; a failed registration leaves nothing behind and can be +/// retried. +fn device_attach_rules_and_registration_faults() { + let mut w = world(); + let (ea, _) = link_up(&mut w); + let deadline = now() + 5 * SECOND; + let missing = w.a.clients[0].grant_device(TO_B, ea, [1; 16], 10, deadline); + assert_eq!(wait(&mut w, &missing).err(), Some(Error::Unavailable)); + let memory = device_memory(); + let wrong_rail = w.a.clients[0].attach_device(3, memory.clone()); + assert_eq!(wait(&mut w, &wrong_rail), Err(Error::InvalidRequest)); + let live = w.sim.live_resources(); + w.sim.fault(Operation::RegisterDmabuf, Fault::Reject); + let failed = w.a.clients[0].attach_device(7, memory.clone()); + assert_eq!(wait(&mut w, &failed), Err(Error::Unavailable)); + assert_eq!(w.sim.live_resources(), live, "nothing registered stays"); + assert_eq!(Arc::strong_count(&memory), 1); + // Memory smaller than one buffer is refused after registration, and + // the region is deregistered again. + let small = w.a.clients[0].attach_device(7, MemfdMemory::new(LEN - 1, 0).unwrap()); + assert_eq!(wait(&mut w, &small), Err(Error::InvalidConfiguration)); + assert_eq!(w.sim.live_resources(), live); + let attached = w.a.clients[0].attach_device(7, memory.clone()); + assert_eq!(wait(&mut w, &attached), Ok(2)); + assert_eq!(w.sim.live_resources(), live + 1); + let again = w.a.clients[0].attach_device(7, device_memory()); + assert_eq!(wait(&mut w, &again), Err(Error::InvalidRequest)); + let attempts: Vec = w + .sim + .trace() + .iter() + .filter(|e| e.operation == Operation::RegisterDmabuf) + .map(|e| e.result) + .collect(); + assert_eq!(attempts, [-1, 0, 0]); + shutdown(w); + assert_eq!(Arc::strong_count(&memory), 1); +} + +#[test] +/// A link break while a device grant is exposed fails the arrival, and the +/// buffer is reused only after the QP is destroyed (the same fence as host +/// grants). +fn device_grant_break_fences_before_reuse() { + let mut w = world(); + let (ea, eb) = link_up(&mut w); + let attached = w.a.clients[0].attach_device(7, device_memory()); + assert_eq!(wait(&mut w, &attached), Ok(2)); + let deadline = now() + 5 * SECOND; + let granted = w.a.clients[0].grant_device(TO_B, ea, [1; 16], 10, deadline); + let granted = wait(&mut w, &granted).unwrap(); + assert_eq!( + w.a.clients[0].device_buffers(7, BufferState::Exposed), + Some(1) + ); + w.sim + .fault_on(Operation::WriteImm, None, Fault::DuplicateImmediate); + let written = w.b.clients[0].write(TO_A, eb, granted.tag, bytes(vec![1; 10]), deadline); + pump(&mut w); + let _ = ready(&written); + assert_eq!(ready(&granted.arrival).err(), Some(Error::Unavailable)); + assert!(matches!(w.a.clients[0].link(TO_B), LinkState::Backoff(_))); + assert_eq!(w.a.clients[0].device_buffers(7, BufferState::Free), Some(2)); + shutdown(w); +} + +#[test] +/// Failed deregistration leaks the device region with its memory owner and +/// charge, so the memory is never freed while the NIC may write it. +fn failed_deregister_leaks_device_memory_owner() { + let mut w = world(); + link_up(&mut w); + let memory = device_memory(); + let attached = w.a.clients[0].attach_device(7, memory.clone()); + assert_eq!(wait(&mut w, &attached), Ok(2)); + w.sim.reject(Operation::Deregister, None, true); + drop(w.a.owner.take()); + assert!(Arc::strong_count(&memory) > 1, "memory owner leaked"); + assert!(w.a.charge.load(Ordering::Acquire) >= 1, "charge leaked"); +} From 4cb3673508c6962dedfe082f97bab0114a819696 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:32:07 +0000 Subject: [PATCH 55/87] feat(racer): back simulated GPU staging with a memfd a dma-buf export can share --- cmd/racer-dataplane/gpu/src/simulation.rs | 88 ++++++++++++++++++++- cmd/racer-dataplane/gpu/tests/simulation.rs | 28 +++++++ 2 files changed, 112 insertions(+), 4 deletions(-) diff --git a/cmd/racer-dataplane/gpu/src/simulation.rs b/cmd/racer-dataplane/gpu/src/simulation.rs index bb6676ff8..cbdb102f7 100644 --- a/cmd/racer-dataplane/gpu/src/simulation.rs +++ b/cmd/racer-dataplane/gpu/src/simulation.rs @@ -163,7 +163,69 @@ struct SimStream { /// Device memory: client memfd or staging bytes. enum Memory { Import { file: File }, - Staging { bytes: Vec, registered: bool }, + Staging { bytes: Mapped, registered: bool }, +} + +/// Staging bytes in a shared memfd mapping. A dma-buf export of the whole +/// buffer is a duplicate of the memfd, so a simulated NIC that maps it writes +/// bytes the simulated GPU then reads, as with real GPUDirect RDMA. +struct Mapped { + file: File, + pointer: std::ptr::NonNull, + length: usize, +} + +impl Mapped { + /// A zeroed shared mapping of `length` bytes, or None. + fn new(length: usize) -> Option { + let fd = unsafe { libc::memfd_create(c"racer-gpu-staging".as_ptr(), libc::MFD_CLOEXEC) }; + if fd < 0 { + return None; + } + let file = unsafe { File::from_raw_fd(fd) }; + file.set_len(length as u64).ok()?; + let pointer = unsafe { + libc::mmap( + std::ptr::null_mut(), + length, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + std::os::fd::AsRawFd::as_raw_fd(&file), + 0, + ) + }; + if pointer == libc::MAP_FAILED { + return None; + } + Some(Self { + file, + pointer: std::ptr::NonNull::new(pointer.cast())?, + length, + }) + } +} + +impl std::ops::Deref for Mapped { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + // SAFETY: the mapping lives as long as self. Other writers (a + // simulated NIC) only write while the dataplane holds the slot. + unsafe { std::slice::from_raw_parts(self.pointer.as_ptr(), self.length) } + } +} + +impl std::ops::DerefMut for Mapped { + fn deref_mut(&mut self) -> &mut [u8] { + // SAFETY: as for deref. + unsafe { std::slice::from_raw_parts_mut(self.pointer.as_ptr(), self.length) } + } +} + +impl Drop for Mapped { + fn drop(&mut self) { + unsafe { libc::munmap(self.pointer.as_ptr().cast(), self.length) }; + } } /// One buffer. @@ -502,6 +564,9 @@ unsafe extern "C" fn staging(context: *mut c_void, size: u64, out: ffi::Out) -> if size > 1 << 32 { return (code::ENOMEM, 0); } + let Some(bytes) = Mapped::new(size as usize) else { + return (code::ENOMEM, 0); + }; let handle = give(w, out); w.buffers.insert( handle, @@ -509,7 +574,7 @@ unsafe extern "C" fn staging(context: *mut c_void, size: u64, out: ffi::Out) -> context: id(context), size, memory: Memory::Staging { - bytes: vec![0; size as usize], + bytes, registered: false, }, }, @@ -813,9 +878,15 @@ unsafe extern "C" fn dmabuf( fd: *mut c_int, ) -> c_int { with(Operation::Dmabuf, |w| { - let Some(size) = w.staging(id(staging)).map(|b| b.len() as u64) else { + let Some(Buffer { + memory: Memory::Staging { bytes, .. }, + .. + }) = w.buffers.get(&id(staging)) + else { return (code::EINVAL, id(staging)); }; + let size = bytes.len() as u64; + let live = (offset == 0).then(|| bytes.file.try_clone()); if fd.is_null() { return (code::EINVAL, id(staging)); } @@ -827,7 +898,16 @@ unsafe extern "C" fn dmabuf( { return (code::ERANGE, id(staging)); } - // Not a live view of the buffer: a placeholder fd of the right size. + // An export from offset 0 is a live view: a duplicate of the backing + // memfd, so the dma-buf offset equals the staging offset as it does on + // real hardware. Other offsets get a placeholder fd of the right size. + if let Some(file) = live { + let Ok(file) = file else { + return (code::EDRIVER, id(staging)); + }; + unsafe { *fd = std::os::fd::IntoRawFd::into_raw_fd(file) }; + return (code::OK, id(staging)); + } let memfd = unsafe { libc::memfd_create(c"racer-gpu-dmabuf".as_ptr(), libc::MFD_CLOEXEC) }; if memfd < 0 { return (code::EDRIVER, id(staging)); diff --git a/cmd/racer-dataplane/gpu/tests/simulation.rs b/cmd/racer-dataplane/gpu/tests/simulation.rs index 147e2eb25..7a4bc3c50 100644 --- a/cmd/racer-dataplane/gpu/tests/simulation.rs +++ b/cmd/racer-dataplane/gpu/tests/simulation.rs @@ -413,6 +413,34 @@ fn dma_buf_export_checks_alignment() { ); } +#[test] +/// A whole-buffer export is a live view: writes through the dma-buf (as a +/// NIC would make) are what the GPU then reads, and the other way around. +fn dma_buf_export_from_zero_is_live() { + use std::os::unix::fs::FileExt; + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let staging = ctx.staging(16384).unwrap(); + let file = std::fs::File::from(staging.dma_buf(0, 16384).unwrap()); + file.write_all_at(b"from the nic", 8192).unwrap(); + let stream = ctx.stream().unwrap(); + let mut out = [0; 12]; + stream.read_staging(&staging, 8192, &mut out).unwrap(); + stream.synchronize().unwrap(); + assert_eq!(&out, b"from the nic"); + stream + .copy_from_host(Destination::Staging(&staging), 4096, b"gpu") + .unwrap(); + stream.synchronize().unwrap(); + let mut back = [0; 3]; + file.read_exact_at(&mut back, 4096).unwrap(); + assert_eq!(&back, b"gpu"); + drop((stream, staging, file)); + assert_eq!(sim.live().staging, 0); +} + #[test] /// Device copies move staged bytes into the window; reads of staging work. fn device_copy_from_staging() { From f1ad90b9670fdcf97c55a9fef2215450b0b6b1fe Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:35:21 +0000 Subject: [PATCH 56/87] feat(racer): GPU landing buffers for RDMA and decrypt from them Add RACER_GPU_LANDING_BYTES. GpuHelper::landing allocates one GPU's landing buffer once, exports it as a dma-buf for rdma_verbs, and keeps it until the helper exits (then leaks it on purpose, since a NIC registration may still cover it). A LandedCiphertext wraps an RDMA device page in that buffer, and the helper decrypts sealed pages from it on the GPU like staged cuFile pages. --- cmd/racer-dataplane/src/config.rs | 32 ++++ cmd/racer-dataplane/src/gpu.rs | 208 +++++++++++++++++++++++- cmd/racer-dataplane/src/read/sealed.rs | 12 ++ cmd/racer-dataplane/src/test_support.rs | 1 + 4 files changed, 249 insertions(+), 4 deletions(-) diff --git a/cmd/racer-dataplane/src/config.rs b/cmd/racer-dataplane/src/config.rs index f409f92e2..61b594baa 100644 --- a/cmd/racer-dataplane/src/config.rs +++ b/cmd/racer-dataplane/src/config.rs @@ -127,6 +127,9 @@ pub struct Config { /// Per-GPU cap on dataplane-owned ciphertext staging memory. Parsed and /// validated now; plaintext `gpu-read` does not allocate staging. pub gpu_staging_bytes: u64, + /// Per-GPU GPU memory that peers' NICs write `gpu-read` ciphertext into + /// over RDMA (GPUDirect RDMA landing). 0 turns landing off. + pub gpu_landing_bytes: u64, /// Opt into experimental opaque HTTP transit; materialized relay is the default. pub opaque_relay: bool, /// Peer-only TCP_NODELAY. Peer exchanges are strictly request/response with @@ -205,6 +208,10 @@ impl Config { { return Err(Error::InvalidConfiguration); } + let gpu_landing_bytes = raw.gpu_landing_bytes.unwrap_or(DEFAULT_GPU_LANDING_BYTES); + if !gpu_landing_bytes.is_multiple_of(GPU_STAGING_UNIT) { + return Err(Error::InvalidConfiguration); + } let mut config = Self { warm: crate::read::warm::Config { max_ttl: Duration::from_secs(raw.warm_max_ttl_secs.unwrap_or(3600)), @@ -259,6 +266,7 @@ impl Config { enable_rdma, enable_gpu, gpu_staging_bytes, + gpu_landing_bytes, opaque_relay: raw.opaque_relay.unwrap_or(false), peer_tcp_nodelay: raw.peer_tcp_nodelay.unwrap_or(true), memory_pressure: raw.memory_pressure.unwrap_or(true), @@ -614,6 +622,9 @@ pub enum GpuMode { /// Default `RACER_GPU_STAGING_BYTES`: 256 MiB per GPU. pub const DEFAULT_GPU_STAGING_BYTES: u64 = 256 * 1024 * 1024; +/// Default `RACER_GPU_LANDING_BYTES`: 256 MiB per GPU. +pub const DEFAULT_GPU_LANDING_BYTES: u64 = 256 * 1024 * 1024; + /// GPU staging is allocated in 2 MiB device granules, so the cap must be a /// whole number of them. pub const GPU_STAGING_UNIT: u64 = 2 * 1024 * 1024; @@ -625,6 +636,7 @@ const ENV_NAMES: &[&str] = &[ "ENABLE_RDMA", "ENABLE_GPU", "GPU_STAGING_BYTES", + "GPU_LANDING_BYTES", "ALLOW_SMT", "OPAQUE_RELAY", "PEER_TCP_NODELAY", @@ -703,6 +715,8 @@ struct RawConfig { enable_gpu: Option, #[serde(default, deserialize_with = "decimal")] gpu_staging_bytes: Option, + #[serde(default, deserialize_with = "decimal")] + gpu_landing_bytes: Option, allow_smt: Option, opaque_relay: Option, peer_tcp_nodelay: Option, @@ -1403,6 +1417,24 @@ mod tests { "{value:?}" ); } + assert_eq!( + parse(&[]).unwrap().gpu_landing_bytes, + DEFAULT_GPU_LANDING_BYTES + ); + for (value, bytes) in [("0", 0), ("4194304", 4 << 20)] { + assert_eq!( + parse(&[("RACER_GPU_LANDING_BYTES", value)]) + .unwrap() + .gpu_landing_bytes, + bytes + ); + } + for value in ["1048576", "3145728", "-2097152", "2MiB"] { + assert!( + parse(&[("RACER_GPU_LANDING_BYTES", value)]).is_err(), + "{value:?}" + ); + } } #[test] diff --git a/cmd/racer-dataplane/src/gpu.rs b/cmd/racer-dataplane/src/gpu.rs index 4c1d85d60..84fcaf90f 100644 --- a/cmd/racer-dataplane/src/gpu.rs +++ b/cmd/racer-dataplane/src/gpu.rs @@ -36,6 +36,18 @@ //! two scratch slots are never shared by two jobs and no staged slot is in use //! by a stream between jobs, which [`racer_gpu::GpuFile::read`] requires. //! +//! # RDMA landing +//! +//! When RDMA is active, each rail may get one GPU's landing buffer +//! ([`GpuHelper::landing`]): separate GPU memory, capped by +//! `RACER_GPU_LANDING_BYTES`, exported once as a dma-buf and registered with +//! the rail's NIC as an `rdma_verbs` device pool. A peer's NIC writes a +//! `gpu-read` page's ciphertext straight into it, and the GPU decrypts from +//! there ([`LandedCiphertext`]). The CPU never reads or copies those bytes. +//! Landing buffers are never freed while the process runs, and are leaked on +//! purpose when the helper exits, because a NIC registration may still cover +//! them. +//! //! At startup the helper probes each GPU with a zero-length decrypt. A GPU //! whose shim has no decrypt kernels, or whose staging cap cannot hold the two //! scratch slots, gets plaintext pages only: the CPU decrypts as for a Get. @@ -168,6 +180,86 @@ impl Drop for StagedCiphertext { } } +/// Each GPU's landing buffer gets its own NIC address range from here. Any +/// value works: the address is local to the memory registration. +const LANDING_IOVA: u64 = 1 << 40; + +/// One GPU's RDMA landing buffer, shared with `rdma_verbs` as a dma-buf. +/// See the module docs, "RDMA landing". +pub struct GpuLanding { + /// Export of the whole landing buffer. + fd: OwnedFd, + ordinal: i32, + length: u64, +} + +impl GpuLanding { + /// The GPU the memory is on. + pub fn ordinal(&self) -> i32 { + self.ordinal + } +} + +impl rdma_verbs::DmabufMemory for GpuLanding { + fn fd(&self) -> std::os::fd::BorrowedFd<'_> { + use std::os::fd::AsFd; + self.fd.as_fd() + } + + fn offset(&self) -> u64 { + 0 + } + + fn length(&self) -> usize { + self.length as usize + } + + fn iova(&self) -> u64 { + LANDING_IOVA * (self.ordinal as u64 + 1) + } +} + +/// Peer ciphertext that a NIC wrote into a GPU landing buffer. Holds the +/// RDMA device page, so the slot is not reused until this is dropped. +pub struct LandedCiphertext { + page: rdma_verbs::DevicePage, + ordinal: i32, +} + +impl LandedCiphertext { + /// Wrap `page` if it lies in `landing`'s memory. + pub fn new(landing: &Arc, page: rdma_verbs::DevicePage) -> Result { + let memory = Arc::as_ptr(page.memory()).cast::<()>(); + if !std::ptr::eq(memory, Arc::as_ptr(landing).cast::<()>()) { + return Err(Error::Internal); + } + Ok(Self { + page, + ordinal: landing.ordinal, + }) + } + + /// The GPU the bytes are on. + pub fn ordinal(&self) -> i32 { + self.ordinal + } + + /// Bytes that landed. + pub fn len(&self) -> u64 { + self.page.len() as u64 + } + + /// Never true: landed ciphertext always holds a tag. + pub fn is_empty(&self) -> bool { + self.page.is_empty() + } + + /// Byte offset in the landing buffer. + fn offset(&self) -> u64 { + self.page.offset() + } +} + /// Counters for GPU decryption, for logs and tests. #[derive(Debug, Default)] pub struct GpuCounters { @@ -185,6 +277,9 @@ pub struct GpuCounters { /// cuFile reads or registrations that failed. pub file_failures: AtomicU64, + + /// Sealed pages decrypted from an RDMA landing buffer. + pub landed: AtomicU64, } /// Per-GPU state shared with I/O threads. @@ -224,6 +319,12 @@ enum Job { id: u64, reply: Option>>, }, + /// Allocate a GPU's RDMA landing buffer and export it as a dma-buf. + Landing { + ordinal: i32, + length: u64, + reply: oneshot::Sender>, + }, #[cfg(test)] Run(Box), } @@ -240,6 +341,8 @@ pub struct GpuHelper { cufile: Arc, counters: Arc, kernels_off: Arc>>, + /// Landing buffers made so far, one per GPU at most. + landings: Mutex>>, } impl GpuHelper { @@ -332,6 +435,7 @@ impl GpuHelper { cufile, counters, kernels_off, + landings: Mutex::new(HashMap::new()), }))) } @@ -404,6 +508,47 @@ impl GpuHelper { answer(&mut receiver, scope).await? } + /// GPU `ordinal`'s RDMA landing buffer of `length` bytes (a multiple of + /// 4 KiB), allocated and exported on first call and reused after that. + /// [`Error::Unavailable`] if the GPU does not decrypt or the export + /// fails, [`Error::Overloaded`] if the GPU has no room. + pub(crate) async fn landing( + &self, + ordinal: i32, + length: u64, + scope: &RequestScope, + ) -> Result> { + if !self.decrypts(ordinal) || length == 0 || !length.is_multiple_of(4096) { + return Err(Error::Unavailable); + } + if let Some(landing) = self + .landings + .lock() + .ok() + .and_then(|l| l.get(&ordinal).cloned()) + { + return Ok(landing); + } + let (reply, mut receiver) = oneshot::channel(); + self.send(Job::Landing { + ordinal, + length, + reply, + })?; + let fd = answer(&mut receiver, scope).await??; + let mut landings = self.landings.lock().map_err(|_| Error::Internal)?; + Ok(landings + .entry(ordinal) + .or_insert_with(|| { + Arc::new(GpuLanding { + fd, + ordinal, + length, + }) + }) + .clone()) + } + /// Where a request on GPU `ordinal` should have sealed pages delivered, /// or None to get plaintext only. pub fn sealed_target(self: &Arc, ordinal: i32) -> Option { @@ -637,6 +782,9 @@ struct Helper { /// long lived, so one registration serves many reads. files: HashMap<(u64, u64), racer_gpu::GpuFile>, requests: HashMap, + /// RDMA landing buffers by GPU. Never freed: a NIC may hold a + /// registration over them. + landing: HashMap, state: HelperState, } @@ -658,6 +806,7 @@ fn helper_main( staging: HashMap::new(), files: HashMap::new(), requests: HashMap::new(), + landing: HashMap::new(), state, } } @@ -675,6 +824,10 @@ fn helper_main( for id in ids { let _ = helper.release(id); } + // A NIC registration may still cover landing memory: never free it. + for (_, buffer) in helper.landing.drain() { + std::mem::forget(buffer); + } } /// Queue a zero-length decrypt to learn whether the shim has working decrypt @@ -819,11 +972,48 @@ impl Helper { let _ = reply.send(result); } } + Job::Landing { + ordinal, + length, + reply, + } => { + let result = self.landing(ordinal, length); + self.note_leaks(); + let _ = reply.send(result); + } #[cfg(test)] Job::Run(f) => f(), } } + /// Allocate GPU `ordinal`'s landing buffer, once, and export it. + fn landing(&mut self, ordinal: i32, length: u64) -> Result { + if !self.landing.contains_key(&ordinal) { + let context = self + .contexts + .iter() + .find(|context| context.ordinal() == ordinal) + .ok_or(Error::Unavailable)?; + let buffer = context.staging(length).map_err(|error| { + eprintln!( + "racer-dataplane: stage=gpu ordinal={ordinal} state=landing-failed reason={error}" + ); + Error::Overloaded + })?; + self.landing.insert(ordinal, buffer); + } + let buffer = self.landing.get(&ordinal).ok_or(Error::Internal)?; + if buffer.size() != length { + return Err(Error::InvalidConfiguration); + } + buffer.dma_buf(0, length).map_err(|error| { + eprintln!( + "racer-dataplane: stage=gpu ordinal={ordinal} state=landing-export-failed reason={error}" + ); + Error::Unavailable + }) + } + /// Try each usable GPU until one owns the allocation. fn import(&mut self, id: u64, fd: OwnedFd, size: u64, window: Range) -> Result { let mut last = racer_gpu::Error::WrongDevice; @@ -953,7 +1143,7 @@ impl Helper { return Err(Error::InvalidRange); } let stream = &request.stream; - let source = match &page.source { + let (source_buffer, source) = match &page.source { SealedSource::Host(copy) => { let bytes = copy.copy.ciphertext.bytes(); if bytes.len() as u64 != length + 16 { @@ -968,13 +1158,20 @@ impl Helper { let _ = stream.synchronize(); return Err(gpu_error(error)); } - CIPHERTEXT_SCRATCH + (staging, CIPHERTEXT_SCRATCH) } SealedSource::Staged { staged, .. } => { if staged.ordinal() != ordinal { return Err(Error::Internal); } - staged.offset() + (staging, staged.offset()) + } + SealedSource::Landed(landed) => { + if landed.ordinal() != ordinal || landed.len() != length + 16 { + return Err(Error::Internal); + } + let buffer = self.landing.get(&ordinal).ok_or(Error::Internal)?; + (buffer, landed.offset()) } }; let request_key = racer_gpu::DecryptRequest { @@ -993,7 +1190,7 @@ impl Helper { } else { (racer_gpu::Destination::Staging(staging), PLAINTEXT_SCRATCH) }; - let queued = stream.decrypt(&request_key, staging, source, destination, at); + let queued = stream.decrypt(&request_key, source_buffer, source, destination, at); let synchronized = stream.synchronize(); if let Err(racer_gpu::Error::KernelUnavailable) = queued { if let Ok(mut off) = self.state.kernels_off.lock() { @@ -1025,6 +1222,9 @@ impl Helper { .counters .decrypted .fetch_add(1, Ordering::Relaxed); + if matches!(page.source, SealedSource::Landed(_)) { + self.state.counters.landed.fetch_add(1, Ordering::Relaxed); + } Ok(()) } diff --git a/cmd/racer-dataplane/src/read/sealed.rs b/cmd/racer-dataplane/src/read/sealed.rs index 8f2e42aa7..ba9e40c2d 100644 --- a/cmd/racer-dataplane/src/read/sealed.rs +++ b/cmd/racer-dataplane/src/read/sealed.rs @@ -14,6 +14,7 @@ use crate::error::Error; use crate::error::Result; use crate::gpu::GpuHelper; +use crate::gpu::LandedCiphertext; use crate::gpu::StagedCiphertext; use crate::memory::PageResult; use crate::memory::UnverifiedPage; @@ -64,6 +65,11 @@ pub enum SealedSource { /// The disk mapping the bytes came from. token: ReadToken, }, + + /// Ciphertext a peer's NIC wrote straight into the target GPU's RDMA + /// landing buffer. No host copy exists, so nothing is shared, published, + /// or persisted, and the page is used once. + Landed(LandedCiphertext), } /// A page that only the GPU will authenticate. Nothing here is proof of @@ -104,6 +110,7 @@ impl SealedPage { let (copy, token) = match &self.source { SealedSource::Host(copy) => (Some(copy.clone()), None), SealedSource::Staged { token, .. } => (None, Some(token.clone())), + SealedSource::Landed(_) => (None, None), }; SealedReport { page: self.page().clone(), @@ -133,6 +140,11 @@ impl SealedPage { { return Err(Error::CorruptRecord); } + if let SealedSource::Landed(landed) = &self.source + && landed.len() != u64::from(self.envelope.ciphertext_length) + { + return Err(Error::CorruptRecord); + } if let SealedSource::Host(copy) = &self.source { let ciphertext = ©.copy.ciphertext; if ciphertext.envelope() != &self.envelope diff --git a/cmd/racer-dataplane/src/test_support.rs b/cmd/racer-dataplane/src/test_support.rs index 8eb3053e9..a9a7fde92 100644 --- a/cmd/racer-dataplane/src/test_support.rs +++ b/cmd/racer-dataplane/src/test_support.rs @@ -312,6 +312,7 @@ pub mod cluster { enable_rdma, enable_gpu: crate::config::GpuMode::Disabled, gpu_staging_bytes: crate::config::DEFAULT_GPU_STAGING_BYTES, + gpu_landing_bytes: crate::config::DEFAULT_GPU_LANDING_BYTES, control_endpoint: "https://control.invalid".into(), peer_listen: "127.0.0.1:0".parse().unwrap(), diagnostics_listen: "127.0.0.1:0".parse().unwrap(), From aa9f79ff5560ef11342b424671e7e4264aa81a63 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:39:57 +0000 Subject: [PATCH 57/87] feat(racer): register GPU landing buffers on RDMA rails and grant into them When GPU decrypt and RDMA are both on, activation gives each rail one GPU's landing buffer (same NUMA node first, then any free GPU) through Client::attach_device. Links::grant_landing grants a slot of that buffer for a gpu-read on the same GPU, and PendingReceive::finish_landed returns the page as LandedCiphertext without touching it on the CPU. Any failure leaves the rail host-only. Adds racer_rdma_pages_landed_total. --- cmd/racer-dataplane/src/app.rs | 11 + cmd/racer-dataplane/src/gpu.rs | 42 ++++ cmd/racer-dataplane/src/rdma/links.rs | 220 +++++++++++++++++++- cmd/racer-dataplane/src/rdma/links/tests.rs | 148 +++++++++++++ cmd/racer-dataplane/src/telemetry.rs | 1 + 5 files changed, 413 insertions(+), 9 deletions(-) diff --git a/cmd/racer-dataplane/src/app.rs b/cmd/racer-dataplane/src/app.rs index 2bf10220a..591b46b1b 100644 --- a/cmd/racer-dataplane/src/app.rs +++ b/cmd/racer-dataplane/src/app.rs @@ -288,6 +288,17 @@ impl Application { self.node.inventory.update(self.discovered_nics.clone())?; Arc::get_mut(&mut self.node).ok_or(Error::Internal)?.gpu = crate::gpu::GpuHelper::start(self.config.enable_gpu, self.config.gpu_staging_bytes)?; + // GPU decrypt and RDMA together: peers' NICs may write gpu-read pages + // straight into GPU landing buffers (GPUDirect RDMA). Activation + // registers them per rail and leaves a rail host-only on any failure. + if self.config.enable_rdma + && let Some(gpu) = &self.node.gpu + { + self.node + .native + .link_node() + .set_gpu_landing(gpu.clone(), self.config.gpu_landing_bytes); + } if self.config.enable_rdma { self.node.native.place(&plan)?; self.node diff --git a/cmd/racer-dataplane/src/gpu.rs b/cmd/racer-dataplane/src/gpu.rs index 84fcaf90f..01fa26cfe 100644 --- a/cmd/racer-dataplane/src/gpu.rs +++ b/cmd/racer-dataplane/src/gpu.rs @@ -1252,3 +1252,45 @@ impl Helper { } } } + +#[cfg(test)] +pub(crate) mod testing { + //! A GPU helper on simulated GPUs, for tests outside the client socket. + use super::*; + use std::cell::RefCell; + + thread_local! { + /// The simulation, on the GPU helper thread only. + static SIM: RefCell> = + const { RefCell::new(None) }; + } + + /// The helper thread's simulation. Call through [`GpuHelper::run`]. + pub(crate) fn simulation() -> racer_gpu::simulation::Simulation { + SIM.with(|sim| sim.borrow().as_ref().unwrap().0.clone()) + } + + /// A required-mode helper on simulated GPUs with these NUMA nodes. + pub(crate) fn helper(numa: &[Option]) -> Arc { + let devices: Vec = numa + .iter() + .enumerate() + .map(|(ordinal, node)| { + let mut device = racer_gpu::simulation::SimDevice::new(ordinal as u8); + device.numa_node = *node; + device + }) + .collect(); + GpuHelper::start_with( + GpuMode::Required, + Box::new(move || { + let simulation = racer_gpu::simulation::Simulation::new(devices); + let environment = simulation.enter(); + SIM.with(|sim| *sim.borrow_mut() = Some((simulation, environment))); + racer_gpu::Gpu::load() + }), + ) + .unwrap() + .unwrap() + } +} diff --git a/cmd/racer-dataplane/src/rdma/links.rs b/cmd/racer-dataplane/src/rdma/links.rs index e2a1d4f9f..4ffa35544 100644 --- a/cmd/racer-dataplane/src/rdma/links.rs +++ b/cmd/racer-dataplane/src/rdma/links.rs @@ -93,6 +93,41 @@ pub struct LinkNode { /// Source of reservation tokens for `initiating`. reservations: std::sync::atomic::AtomicU64, + + /// GPU helper and per-GPU landing bytes, when `gpu-read` pages may land + /// in GPU memory (GPUDirect RDMA). + gpu_landing: Mutex, u64)>>, + + /// The GPU landing buffer registered on each rail this round. + landings: Mutex>>, +} + +/// Pick at most one GPU per rail for landing, and each GPU for at most one +/// rail. A rail first takes a free GPU on its NUMA node, then any free GPU. +/// `rails` and `gpus` give each one's NUMA node, if known. PCI distance is +/// not used: only NUMA locality is known on both sides. +pub fn assign_landing( + rails: &[(RailId, Option)], + gpus: &[(i32, Option)], +) -> Vec<(RailId, i32)> { + let mut free: Vec<(i32, Option)> = gpus.to_vec(); + let mut assigned = Vec::new(); + let mut later = Vec::new(); + for &(rail, numa) in rails { + match free + .iter() + .position(|&(_, node)| numa.is_some() && node == numa) + { + Some(index) => assigned.push((rail, free.remove(index).0)), + None => later.push(rail), + } + } + for rail in later { + if !free.is_empty() { + assigned.push((rail, free.remove(0).0)); + } + } + assigned } /// One setup this node initiated: reserved before the owner answers, then @@ -150,7 +185,20 @@ impl LinkNode { .store(true, std::sync::atomic::Ordering::Release); } + /// Let `gpu-read` pages land in GPU memory: on each activation, give + /// each rail one GPU's landing buffer of `bytes` (see + /// [`assign_landing`]). 0 bytes turns landing off. + pub fn set_gpu_landing(&self, helper: Arc, bytes: u64) { + *Self::lock(&self.gpu_landing) = (bytes != 0).then_some((helper, bytes)); + } + + /// The GPU landing buffer on `rail` this round, if any. + pub fn landing(&self, rail: RailId) -> Option> { + Self::lock(&self.landings).get(&rail.0).cloned() + } + fn reset(&self) { + Self::lock(&self.landings).clear(); Self::lock(&self.active).clear(); self.settled .store(false, std::sync::atomic::Ordering::Release); @@ -343,6 +391,9 @@ impl Links { let result = self .activate_inner(publication, admission, registered_bytes, scope) .await; + if let Ok(rails) = &result { + self.attach_landings(rails, scope).await; + } match &result { Ok(rails) => self.node.settle(rails.clone()), Err(Error::Cancelled | Error::DeadlineExceeded) => {} @@ -431,6 +482,62 @@ impl Links { .collect()) } + /// Register GPU landing buffers on the activated `rails`. Failures only + /// log: pages then land in host memory as usual. + async fn attach_landings(&self, rails: &[RailMapping], scope: &RequestScope) { + let Some((helper, bytes)) = LinkNode::lock(&self.node.gpu_landing).clone() else { + return; + }; + if bytes < BUFFER_LEN as u64 { + eprintln!( + "racer-dataplane: stage=gpu-landing state=disabled reason=too-small bytes={bytes}" + ); + return; + } + let gpus: Vec<(i32, Option)> = helper + .devices() + .iter() + .filter(|device| helper.decrypts(device.ordinal)) + .map(|device| (device.ordinal, device.numa_node.map(|n| n as usize))) + .collect(); + let rails: Vec<(RailId, Option)> = + rails.iter().map(|m| (m.rail, m.numa_node)).collect(); + for (rail, ordinal) in assign_landing(&rails, &gpus) { + let landing = match helper.landing(ordinal, bytes, scope).await { + Ok(landing) => landing, + Err(error) => { + eprintln!( + "racer-dataplane: stage=gpu-landing rail={} ordinal={ordinal} state=failed reason={error}", + rail.0 + ); + continue; + } + }; + let memory = landing.clone() as rdma_verbs::DeviceMemory; + match self + .wait(self.client.attach_device(rail.0, memory), scope) + .await + { + Ok(slots) => { + eprintln!( + "racer-dataplane: stage=gpu-landing rail={} ordinal={ordinal} state=enabled slots={slots}", + rail.0 + ); + LinkNode::lock(&self.node.landings).insert(rail.0, landing); + } + Err(error) => eprintln!( + "racer-dataplane: stage=gpu-landing rail={} ordinal={ordinal} state=failed reason={error}", + rail.0 + ), + } + } + } + + /// The GPU whose landing buffer is on `rail`, if any. + pub fn landing_ordinal(&self, rail: RailId) -> Option { + self.node.landing(rail).map(|landing| landing.ordinal()) + } + /// Close the node: every link breaks and the owner drains. A later /// [`Self::activate`] reopens once drained. pub fn close(&self) { @@ -682,6 +789,52 @@ impl Links { rail, transfer, arrival: granted.arrival, + landing: None, + }) + } + + /// Receiver: like [`Self::grant`], but the window is over a slot of the + /// GPU landing buffer on `rail`, which must be on GPU `ordinal`. Fails + /// with `Unavailable` if it is not, and `Overloaded` if no slot is free; + /// the caller then grants host memory. + pub async fn grant_landing( + &self, + peer: &NodeId, + rail: RailId, + transfer: TransferId, + ordinal: i32, + scope: &RequestScope, + ) -> Result { + scope.check()?; + let landing = self + .node + .landing(rail) + .filter(|landing| landing.ordinal() == ordinal) + .ok_or(Error::Unavailable)?; + let (epoch, link) = self.ready(peer, rail).ok_or(Error::Unavailable)?; + let length = MAX_CIPHERTEXT as u32; + let granted = self + .wait( + self.client.grant_device( + link_key(peer, rail), + epoch, + transfer.0, + length, + scope.deadline.0, + ), + scope, + ) + .await?; + Ok(PendingReceive { + metrics: self.metrics.clone(), + retry: self.retry.clone(), + link, + tag: granted.tag, + length, + rail, + transfer, + arrival: granted.arrival, + landing: Some(landing), }) } @@ -791,9 +944,65 @@ pub struct PendingReceive { rail: RailId, transfer: TransferId, arrival: Op, + + /// The GPU landing buffer the grant is over, for a landing grant. + landing: Option>, } impl PendingReceive { + /// True for a grant over a GPU landing slot ([`Links::grant_landing`]). + pub fn is_landing(&self) -> bool { + self.landing.is_some() + } + + /// Check the signed completion against this grant and the envelope. + fn check_completion(&self, head: &VerifiedHead, envelope: &PageEnvelope) -> Result { + validate_envelope(envelope)?; + let length = envelope.ciphertext_length; + if signed_value(head, COMPLETION_HEADER, COMPLETE_LEN)? + != completion_value(self.link, self.tag, length, self.transfer) + || length > self.length + { + return Err(Error::Unauthorized); + } + Ok(length) + } + + /// Like [`Self::finish`] for a landing grant: the ciphertext stays in GPU + /// memory and comes back as [`crate::gpu::LandedCiphertext`]. The CPU + /// never reads it; the GPU checks the tag before writing plaintext. + pub async fn finish_landed( + self, + head: &VerifiedHead, + envelope: &PageEnvelope, + scope: &RequestScope, + ) -> Result { + scope.check()?; + let length = self.check_completion(head, envelope)?; + let landing = self.landing.clone().ok_or(Error::Internal)?; + let arrival = self + .retry + .wait(scope, poll_fn(|cx| self.arrival.poll(cx))) + .await; + let page = match arrival { + Ok(Arrival::Device(page)) => page, + Ok(Arrival::Released | Arrival::Page(_)) => return Err(Error::Io), + Err(error @ Error::Cancelled) => return Err(error), + Err(_) => { + scope.check()?; + return Err(Error::Io); + } + }; + if page.len() != length as usize { + return Err(Error::InvalidRange); + } + if let Some(metrics) = &self.metrics { + metrics.record(crate::telemetry::Event::RdmaPagesReceived, 1); + metrics.record(crate::telemetry::Event::RdmaReceivedBytes, length.into()); + metrics.record(crate::telemetry::Event::RdmaPagesLanded, 1); + } + crate::gpu::LandedCiphertext::new(&landing, page) + } /// Base64 value of the signed `GRANT_HEADER` extension. pub fn header_value(&self) -> Vec { STANDARD @@ -834,14 +1043,7 @@ impl PendingReceive { scope: &RequestScope, ) -> Result { scope.check()?; - validate_envelope(&envelope)?; - let length = envelope.ciphertext_length; - if signed_value(head, COMPLETION_HEADER, COMPLETE_LEN)? - != completion_value(self.link, self.tag, length, self.transfer) - || length > self.length - { - return Err(Error::Unauthorized); - } + let length = self.check_completion(head, &envelope)?; let reservation = admission.reserve( Some(&envelope.page.version.object.cache), ResourceClass::Ciphertext, @@ -853,7 +1055,7 @@ impl PendingReceive { .await; let page = match arrival { Ok(Arrival::Page(page)) => page, - // Only `grant` reaches here, and it never asks for a device page. + // A landing grant finishes with `finish_landed`. Ok(Arrival::Released | Arrival::Device(_)) => return Err(Error::Io), Err(error @ Error::Cancelled) => return Err(error), Err(_) => { diff --git a/cmd/racer-dataplane/src/rdma/links/tests.rs b/cmd/racer-dataplane/src/rdma/links/tests.rs index 22761751c..f37866fe6 100644 --- a/cmd/racer-dataplane/src/rdma/links/tests.rs +++ b/cmd/racer-dataplane/src/rdma/links/tests.rs @@ -847,3 +847,151 @@ mod send_in_place { assert_eq!(ma.count(Event::RdmaHttpFallback), 1); } } + +#[test] +/// Rails take a free GPU on their NUMA node first, then any free GPU; each +/// GPU serves one rail at most. +fn assign_landing_prefers_numa_then_any() { + let rails = [ + (RailId(1), Some(1)), + (RailId(2), Some(0)), + (RailId(3), None), + (RailId(4), Some(0)), + ]; + let gpus = [(0, Some(0)), (1, Some(1)), (2, None)]; + assert_eq!( + assign_landing(&rails, &gpus), + [(RailId(1), 1), (RailId(2), 0), (RailId(3), 2)] + ); + assert_eq!(assign_landing(&rails, &[]), []); + assert_eq!( + assign_landing(&[(RailId(9), Some(3))], &[(5, Some(1))]), + [(RailId(9), 5)] + ); +} + +/// Register a simulated GPU's landing buffer on `a`'s rail. +fn attach_landing(w: &mut World, helper: &Arc, bytes: u64) { + w.a.links.node().set_gpu_landing(helper.clone(), bytes); + let rails = w.a.links.active(); + let scope = scope(); + // The GPU helper answers from its own thread: poll with short sleeps. + let mut future = std::pin::pin!(w.a.links.attach_landings(&rails, &scope)); + let mut cx = Context::from_waker(futures::task::noop_waker_ref()); + for _ in 0..5000 { + if future.as_mut().poll(&mut cx).is_ready() { + return; + } + pump(&mut w.a.owner, &mut w.b.owner); + std::thread::sleep(Duration::from_millis(1)); + } + panic!("landing attach did not finish"); +} + +#[test] +/// A landing grant takes a slot of the GPU landing buffer: the peer's write +/// lands in GPU memory, the CPU path never sees it, and the slot is held +/// until the landed page drops. A landing grant for another GPU, or with no +/// free slot, is refused so the caller grants host memory instead. +fn landing_grant_receives_into_gpu_memory() { + use std::os::unix::fs::FileExt; + let helper = crate::gpu::testing::helper(&[None]); + let mut w = world(); + let link = link_up(&mut w); + // One slot only. + let bytes = (BUFFER_LEN as u64).next_multiple_of(crate::config::GPU_STAGING_UNIT); + attach_landing(&mut w, &helper, bytes); + assert_eq!(w.a.links.landing_ordinal(RAIL), Some(0)); + assert_eq!(w.b.links.landing_ordinal(RAIL), None); + let (a, b) = (node_id(1), node_id(2)); + let scope = scope(); + assert_eq!( + drive!( + w, + w.a.links + .grant_landing(&b, RAIL, TransferId([3; 16]), 1, &scope) + ) + .err(), + Some(Error::Unavailable), + "landing buffer is on another GPU" + ); + let pending = drive!( + w, + w.a.links + .grant_landing(&b, RAIL, TransferId([3; 16]), 0, &scope) + ) + .unwrap(); + assert!(pending.is_landing()); + assert_eq!(pending.link(), link); + assert!( + drive!( + w, + w.a.links + .grant_landing(&b, RAIL, TransferId([4; 16]), 0, &scope) + ) + .is_err(), + "the only slot is taken" + ); + let header = STANDARD.decode(pending.header_value()).unwrap(); + let grant = AuthenticatedGrant::decode(&header, RAIL, TransferId([3; 16])).unwrap(); + let page = page(&w.b.admission.clone(), 35, 0x6b); + drive!(w, w.b.links.send(&a, &grant, &page, &scope)).unwrap(); + let device = match drive!(w, poll_fn(|cx| pending.arrival.poll(cx))).unwrap() { + Arrival::Device(page) => page, + _ => panic!("expected a device page"), + }; + // The bytes are in the GPU buffer, read here through its dma-buf. + let landing = w.a.links.node().landing(RAIL).unwrap(); + let mut bytes = [0; 35]; + let file = std::fs::File::from( + rdma_verbs::DmabufMemory::fd(&*landing) + .try_clone_to_owned() + .unwrap(), + ); + file.read_exact_at(&mut bytes, device.offset()).unwrap(); + assert_eq!(bytes, [0x6b; 35]); + let landed = crate::gpu::LandedCiphertext::new(&landing, device).unwrap(); + assert_eq!((landed.ordinal(), landed.len()), (0, 35)); + // The host pool was not used. + assert_eq!(w.a.links.buffers(RAIL, BufferState::Free), Some(2)); + drop(landed); + pump!(w); + let again = drive!( + w, + w.a.links + .grant_landing(&b, RAIL, TransferId([5; 16]), 0, &scope) + ) + .unwrap(); + let header = STANDARD.decode(again.header_value()).unwrap(); + let grant = AuthenticatedGrant::decode(&header, RAIL, TransferId([5; 16])).unwrap(); + w.b.links.release(&a, &grant); + assert!(matches!( + drive!(w, poll_fn(|cx| again.arrival.poll(cx))), + Ok(Arrival::Released) + )); + // Closing the node forgets the landing buffer for the next round. + w.a.links.close(); + assert_eq!(w.a.links.landing_ordinal(RAIL), None); +} + +#[test] +/// No usable landing: too little memory, no GPU, or a failed registration +/// leaves the rail host-only. +fn landing_failures_leave_rails_host_only() { + let helper = crate::gpu::testing::helper(&[None]); + let mut w = world(); + attach_landing(&mut w, &helper, (BUFFER_LEN as u64) / 2); + assert_eq!(w.a.links.landing_ordinal(RAIL), None); + attach_landing(&mut w, &helper, 0); + assert_eq!(w.a.links.landing_ordinal(RAIL), None); + w._sim.fault( + rdma_verbs::simulation::Operation::RegisterDmabuf, + rdma_verbs::simulation::Fault::Reject, + ); + let bytes = (BUFFER_LEN as u64).next_multiple_of(crate::config::GPU_STAGING_UNIT); + attach_landing(&mut w, &helper, bytes); + assert_eq!(w.a.links.landing_ordinal(RAIL), None); + // Registration works on the next try, reusing the same GPU buffer. + attach_landing(&mut w, &helper, bytes); + assert_eq!(w.a.links.landing_ordinal(RAIL), Some(0)); +} diff --git a/cmd/racer-dataplane/src/telemetry.rs b/cmd/racer-dataplane/src/telemetry.rs index 4852561b6..6b2fe9421 100644 --- a/cmd/racer-dataplane/src/telemetry.rs +++ b/cmd/racer-dataplane/src/telemetry.rs @@ -3310,6 +3310,7 @@ metrics! { Event, EVENTS, EVENT_COUNT; Self::RdmaSentBytes => "racer_rdma_sent_bytes_total", Self::RdmaPagesReceived => "racer_rdma_pages_received_total", Self::RdmaReceivedBytes => "racer_rdma_received_bytes_total", + Self::RdmaPagesLanded => "racer_rdma_pages_landed_total", Self::RdmaHttpFallback => "racer_rdma_http_fallback_total", Self::RdmaGrantCount => "racer_rdma_grant_nanoseconds_count", Self::RdmaGrantNs => "racer_rdma_grant_nanoseconds_sum", From fdf0b571059b2db453bb65b4052c5d6aed5566d7 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:42:31 +0000 Subject: [PATCH 58/87] feat(racer): receive a requested peer page into a GPU landing buffer over RDMA Requester::request_landing threads a gpu-read's GPU ordinal down to the peer transport. When the hop is RDMA and the rail's landing buffer is on that GPU, accept_native grants a landing slot (else host memory as usual). A completed landing write comes back as RelayResponse::Landed, whose signed head is checked for exactly this page (landed_response_head) and verified against the request binding (Forwarding::verify_landed). --- cmd/racer-dataplane/src/peer.rs | 117 ++++++++++++- cmd/racer-dataplane/src/peer/forwarding.rs | 22 +++ cmd/racer-dataplane/src/peer/protocol.rs | 16 ++ cmd/racer-dataplane/src/peer/server.rs | 4 + cmd/racer-dataplane/src/peer/tests.rs | 3 + .../src/peer/tests/protocol_socket.rs | 1 + cmd/racer-dataplane/src/peer/transport.rs | 158 ++++++++++++++++-- 7 files changed, 301 insertions(+), 20 deletions(-) diff --git a/cmd/racer-dataplane/src/peer.rs b/cmd/racer-dataplane/src/peer.rs index 3b6fd6da9..8e10fd944 100644 --- a/cmd/racer-dataplane/src/peer.rs +++ b/cmd/racer-dataplane/src/peer.rs @@ -424,6 +424,8 @@ impl Relay { .append_response(response, &previous) .map(transport::RelayResponse::Complete) } + // A relay never asks for landing. + transport::RelayResponse::Landed(_) => Err(Error::Internal), transport::RelayResponse::Http { authentication, connection, @@ -444,6 +446,16 @@ impl Relay { } } +/// What [`Requester::request_landing`] produced. +#[allow(clippy::large_enum_variant)] // Short-lived, returned once per request. +pub(crate) enum Delivered { + /// A verified response, as from [`Requester::request`]. + Response(VerifiedResponse), + + /// A verified page in a GPU landing buffer. + Landed(Box), +} + /// Concrete candidate requester. Scripted outcomes exist only in unit tests and /// the subscription interoperability fixture, so cancellation and an adversarial /// late-success completion fence remain distinct. @@ -677,7 +689,15 @@ impl Requester { let (signed, binding) = forwarding.sign_request_to(request, &next)?; let mut timing = PageTiming::new(metrics); let response = self - .exchange_inner_mode(signed, membership, None, &scope, true, Some(&mut timing)) + .exchange_inner_mode( + signed, + membership, + None, + &scope, + true, + Some(&mut timing), + None, + ) .await?; let transport::RelayResponse::Complete(response) = response else { return Err(Error::Internal); @@ -698,6 +718,58 @@ impl Requester { membership: std::sync::Arc, scope: &'a RequestScope, ) -> Operation<'a, VerifiedResponse> { + Box::pin(async move { + match self + .request_network_landing(request, membership, None, scope) + .await? + { + Delivered::Response(response) => Ok(response), + Delivered::Landed(_) => Err(Error::Internal), + } + }) + } + + /// Like [`Self::request`] for one fixed page that a `gpu-read` will + /// decrypt on GPU `ordinal`. If the last hop is RDMA and this node has a + /// free landing slot on that GPU's rail, the page lands in GPU memory and + /// comes back as [`Delivered::Landed`]. Otherwise this is a plain request. + pub(crate) fn request_landing<'a>( + &'a self, + request: PeerRequest, + membership: std::sync::Arc, + ordinal: i32, + scope: &'a RequestScope, + ) -> Operation<'a, Delivered> { + match self { + Self::Network { .. } => { + self.request_network_landing(request, membership, Some(ordinal), scope) + } + #[cfg(any(test, feature = "subscription-interop"))] + Self::Scripted { + request: script, .. + } => { + let response = script(request, membership, scope.clone(), false); + Box::pin(async move { response.await.map(Delivered::Response) }) + } + } + } + + /// Whether this node has a landing buffer for GPU `ordinal` on any rail. + pub(crate) fn landing_ready(&self, ordinal: i32) -> bool { + match self { + Self::Network { transfers, .. } => transfers.landing_ready(ordinal), + #[cfg(any(test, feature = "subscription-interop"))] + Self::Scripted { .. } => false, + } + } + + fn request_network_landing<'a>( + &'a self, + request: PeerRequest, + membership: std::sync::Arc, + landing: Option, + scope: &'a RequestScope, + ) -> Operation<'a, Delivered> { let (network, paths, observer, forwarding, metrics) = match self { Self::Network { network, @@ -727,10 +799,24 @@ impl Requester { let (signed, binding) = forwarding.sign_request_to(request, next)?; let mut timing = PageTiming::new(metrics); let response = self - .exchange_inner_mode(signed, membership, None, &scope, false, Some(&mut timing)) + .exchange_inner_mode( + signed, + membership, + None, + &scope, + false, + Some(&mut timing), + landing, + ) .await?; - let transport::RelayResponse::Complete(response) = response else { - return Err(Error::Internal); + let response = match response { + transport::RelayResponse::Complete(response) => response, + // Verified by exchange_inner_mode against `binding`. + transport::RelayResponse::Landed(landed) => { + scope.check()?; + return Ok(Delivered::Landed(landed)); + } + transport::RelayResponse::Http { .. } => return Err(Error::Internal), }; scope.check()?; let response = observer.result( @@ -741,7 +827,7 @@ impl Requester { .await, )?; timing.success(&response); - Ok(response) + Ok(Delivered::Response(response)) }) } } @@ -782,7 +868,7 @@ impl Requester { relay: Option>>, scope: &'a RequestScope, ) -> Operation<'a, transport::RelayResponse> { - self.exchange_inner_mode(request, membership, relay, scope, false, None) + self.exchange_inner_mode(request, membership, relay, scope, false, None, None) } fn exchange_inner_mode<'a>( &'a self, @@ -792,6 +878,7 @@ impl Requester { scope: &'a RequestScope, direct_http: bool, timing: Option<&'a mut PageTiming<'_>>, + landing: Option, ) -> Operation<'a, transport::RelayResponse> { let (network, paths, health, forwarding, transfers) = match self { Self::Network { @@ -877,6 +964,7 @@ impl Requester { receive_permit, socket_failure.clone(), timing, + landing, &scope, ) .await; @@ -888,7 +976,9 @@ impl Requester { }; use crate::topology::LinkOutcome; let outcome = match &response { - Ok(transport::RelayResponse::Complete(_)) => Some(LinkOutcome::Success), + Ok(transport::RelayResponse::Complete(_) | transport::RelayResponse::Landed(_)) => { + Some(LinkOutcome::Success) + } Err(_) if permit.is_none() && socket_failure.get() => Some(LinkOutcome::Refused), // Unavailable/deadline/protocol errors can arise locally or at a // downstream node. Do not blame an immediate peer without evidence. @@ -906,7 +996,9 @@ impl Requester { { Outcome::Neutral } - Ok(transport::RelayResponse::Complete(_)) => Outcome::Verified, + Ok( + transport::RelayResponse::Complete(_) | transport::RelayResponse::Landed(_), + ) => Outcome::Verified, Err(Error::Overloaded) => Outcome::LocalPressure, _ => Outcome::Neutral, }); @@ -934,6 +1026,15 @@ impl Requester { .verify_response_async(response, binding, scope) .await .map(|verified| transport::RelayResponse::Complete(verified.into_signed())), + transport::RelayResponse::Landed(landed) => { + forwarding.verify_landed( + &landed.authentication, + &landed.metadata, + &landed.envelope, + binding, + )?; + Ok(transport::RelayResponse::Landed(landed)) + } transport::RelayResponse::Http { authentication, mut connection, diff --git a/cmd/racer-dataplane/src/peer/forwarding.rs b/cmd/racer-dataplane/src/peer/forwarding.rs index ce97e15af..a205d3ed4 100644 --- a/cmd/racer-dataplane/src/peer/forwarding.rs +++ b/cmd/racer-dataplane/src/peer/forwarding.rs @@ -576,6 +576,28 @@ impl Forwarding { let (origin, forwarders) = self.verify_response_parts(&response, request)?; Ok(Self::finish_response(response, request, origin, forwarders)) } + /// Like [`Self::verify_response`] for a page that landed in GPU memory: + /// verify the signed head for `metadata` and `envelope`. The GPU checks + /// the ciphertext itself with the AEAD tag. + pub(crate) fn verify_landed( + &self, + authentication: &ForwardedHead, + metadata: &crate::model::ObjectMetadata, + envelope: &crate::model::PageEnvelope, + request: &RequestBinding, + ) -> Result<()> { + let path = protocol::decode_nodes( + field(&authentication.original.head, "racer-response-path")?.as_bytes(), + )?; + let expected = protocol::landed_response_head( + metadata, + envelope, + &signed_digest(&request.original)?, + &path, + )?; + self.verify_response_head(authentication, &expected, request) + .map(|_| ()) + } fn verify_response_parts( &self, response: &SignedResponse, diff --git a/cmd/racer-dataplane/src/peer/protocol.rs b/cmd/racer-dataplane/src/peer/protocol.rs index e11a322ec..2145d4b2a 100644 --- a/cmd/racer-dataplane/src/peer/protocol.rs +++ b/cmd/racer-dataplane/src/peer/protocol.rs @@ -768,6 +768,22 @@ pub fn response_head( Ok(head) } +/// The response head for a page whose ciphertext a NIC wrote straight into +/// GPU landing memory. It matches [`response_head`] for +/// [`PeerResponse::Page`]; only the metadata and envelope are signed, never +/// the ciphertext bytes, so no host copy is needed to check it. +pub(crate) fn landed_response_head( + metadata: &ObjectMetadata, + envelope: &crate::model::PageEnvelope, + request_digest: &[u8; 32], + path: &[NodeId], +) -> Result { + let mut head = response_prefix(request_digest, path)?; + page_fields(&mut head, metadata, envelope)?; + response_tail(&mut head, "page", u64::from(envelope.ciphertext_length)); + Ok(head) +} + /// Start a canonical response with its retained request binding and reverse path. fn response_prefix(binding: &[u8; 32], path: &[NodeId]) -> Result { let mut head = MessageHead { diff --git a/cmd/racer-dataplane/src/peer/server.rs b/cmd/racer-dataplane/src/peer/server.rs index a3b287c87..0eb85eda6 100644 --- a/cmd/racer-dataplane/src/peer/server.rs +++ b/cmd/racer-dataplane/src/peer/server.rs @@ -519,6 +519,10 @@ impl PeerServer { return result; } Ok((super::transport::RelayResponse::Complete(response), _)) => response, + // A relay never asks for landing. + Ok((super::transport::RelayResponse::Landed(_), _)) => { + return Err(Error::Internal); + } Err(Error::Overloaded) => self .forwarding .sign_response(&binding, PeerResponse::Overloaded)?, diff --git a/cmd/racer-dataplane/src/peer/tests.rs b/cmd/racer-dataplane/src/peer/tests.rs index 141a5e4cf..dd8d37617 100644 --- a/cmd/racer-dataplane/src/peer/tests.rs +++ b/cmd/racer-dataplane/src/peer/tests.rs @@ -351,6 +351,7 @@ mod body_progress { None, Rc::new(Cell::new(false)), Some(&mut timing), + None, &scope, ); let result = loop { @@ -1309,6 +1310,7 @@ mod encrypted_http { None, Rc::new(Cell::new(false)), None, + None, turn, ) .await; @@ -1708,6 +1710,7 @@ mod requester_safety { matches!(response.response, PeerResponse::Overloaded), case == "overloaded" ), + RelayResponse::Landed(_) => panic!("nothing asked for landing"), } } assert_eq!(metrics.count(Event::PeerProbe), u64::from(case != "direct")); diff --git a/cmd/racer-dataplane/src/peer/tests/protocol_socket.rs b/cmd/racer-dataplane/src/peer/tests/protocol_socket.rs index 7b5f4f2b1..c4affa246 100644 --- a/cmd/racer-dataplane/src/peer/tests/protocol_socket.rs +++ b/cmd/racer-dataplane/src/peer/tests/protocol_socket.rs @@ -1396,6 +1396,7 @@ fn real_http_ciphertext_fragmentation_pool_reuse_and_truncation() { None, Rc::new(std::cell::Cell::new(false)), None, + None, &scope, ) .await; diff --git a/cmd/racer-dataplane/src/peer/transport.rs b/cmd/racer-dataplane/src/peer/transport.rs index 105a2ec78..19abee990 100644 --- a/cmd/racer-dataplane/src/peer/transport.rs +++ b/cmd/racer-dataplane/src/peer/transport.rs @@ -403,13 +403,27 @@ impl Transfers { .await?; Ok((written.lease, true)) } + /// Whether a rail of this node has a landing buffer on GPU `ordinal`. + pub(crate) fn landing_ready(&self, ordinal: i32) -> bool { + self.links.as_ref().is_some_and(|links| { + links + .active() + .iter() + .any(|rail| links.landing_ordinal(rail.rail) == Some(ordinal)) + }) + } /// Receiver: when the hop plan is RDMA and the link is ready, grant a page /// window in-band and sign it into the request's `Accept` control. Any /// recoverable failure means a plain HTTP request. + /// + /// With `landing`, the GPU of a `gpu-read` that asked for this fixed page, + /// the grant is over a slot of that GPU's landing buffer when this rail + /// has one and a slot is free. Otherwise it is a host grant as usual. pub(super) async fn accept_native( &self, request: &SignedRequest, plan: TransportPlan, + landing: Option, scope: &RequestScope, ) -> Result> { let TransportPlan::Rdma { rail } = plan else { @@ -436,10 +450,34 @@ impl Transfers { scope, rail, )?; - let pending = match links - .grant(&peer, rail, binding.transfer, &native_scope(scope)) - .await - { + let landed = match landing { + Some(ordinal) + if matches!( + request.request.operation, + crate::peer::protocol::Operation::Page { .. } + ) && links.landing_ordinal(rail) == Some(ordinal) => + { + match links + .grant_landing(&peer, rail, binding.transfer, ordinal, &native_scope(scope)) + .await + { + Ok(pending) => Some(pending), + // No free landing slot: grant host memory instead. + Err(error) if native_failure(error, scope) => None, + Err(error) => return Err(error), + } + } + _ => None, + }; + let granted = match landed { + Some(pending) => Ok(pending), + None => { + links + .grant(&peer, rail, binding.transfer, &native_scope(scope)) + .await + } + }; + let pending = match granted { Ok(pending) => pending, Err(error) if native_failure(error, scope) => { links.record_http_fallback(); @@ -525,12 +563,13 @@ impl Transfers { /// Receiver: read the sender's one signed outcome. `Complete` means the page /// is in the granted window; anything after that is `Io`, because the HTTP /// body was never sent. `Finish` carries the HTTP body. + #[cfg(test)] #[allow(clippy::too_many_arguments)] // Preserve explicit authenticated transfer inputs. pub(super) async fn receive_native( &self, connection: ConnectionLease, authentication: ForwardedHead, - mut binding: Binding, + binding: Binding, accept: SignedHead, peer: NodeId, control: SignedHead, @@ -539,6 +578,43 @@ impl Transfers { length: usize, scope: &RequestScope, ) -> Result { + match self + .receive_native_into( + connection, + authentication, + binding, + accept, + peer, + control, + pending, + membership, + length, + scope, + ) + .await? + { + RelayResponse::Complete(response) => Ok(response), + _ => Err(Error::Internal), + } + } + + /// Like [`Self::receive_native`], but a landing grant + /// ([`Self::accept_native`] with a GPU) whose page the sender wrote + /// comes back as [`RelayResponse::Landed`]. + #[allow(clippy::too_many_arguments)] // Preserve explicit authenticated transfer inputs. + pub(super) async fn receive_native_into( + &self, + connection: ConnectionLease, + authentication: ForwardedHead, + mut binding: Binding, + accept: SignedHead, + peer: NodeId, + control: SignedHead, + pending: PendingReceive, + membership: &std::sync::Arc, + length: usize, + scope: &RequestScope, + ) -> Result { binding.response = envelope_digest(&authentication)?; let (verified, phase) = binding.verify( &self.signatures, @@ -572,7 +648,49 @@ impl Transfers { drop(pending); return self .receive_fallback(connection, authentication, length, scope) - .await; + .await + .map(RelayResponse::Complete); + } + if pending.is_landing() { + let landed = pending + .finish_landed(&verified, &envelope, &native_scope(scope)) + .await?; + #[cfg(test)] + self.native_completions + .set(self.native_completions.get() + 1); + // Only a fixed page request asks for landing. The signed head must + // describe exactly this page; the requester verifies the chain + // against its binding, and the GPU checks the tag. + let original = &authentication.original.head; + if crate::peer::protocol::field(original, "racer-outcome")? != "page" { + return Err(Error::Unauthorized); + } + let request_digest = crate::peer::protocol::decode_binary( + crate::peer::protocol::field(original, "racer-request-binding")?.as_bytes(), + )? + .try_into() + .map_err(|_| Error::InvalidRequest)?; + let path = crate::peer::protocol::decode_nodes( + crate::peer::protocol::field(original, "racer-response-path")?.as_bytes(), + )?; + crate::peer::protocol::agrees( + original, + &crate::peer::protocol::landed_response_head( + &metadata, + &envelope, + &request_digest, + &path, + )?, + false, + )?; + let mut connection = connection; + connection.finish_exchange()?; + return Ok(RelayResponse::Landed(Box::new(LandedResponse { + authentication, + metadata, + envelope, + landed, + }))); } let (admission, _) = &self.wire; // The arrival stays in registered memory; retaining paths copy it out. @@ -615,10 +733,10 @@ impl Transfers { )?; let mut connection = connection; connection.finish_exchange()?; - Ok(SignedResponse { + Ok(RelayResponse::Complete(SignedResponse { authentication, response, - }) + })) } async fn receive_fallback( &self, @@ -964,12 +1082,28 @@ fn observe_read( result } +/// A peer page whose ciphertext a NIC wrote into a GPU landing buffer, with +/// its signed response envelope. The requester verifies the signatures +/// against its outstanding binding, as for any response, and the GPU checks +/// the tag. There are no host bytes: this never reaches memory, disk, or +/// another reader. +pub struct LandedResponse { + pub authentication: crate::peer::forwarding::ForwardedHead, + pub metadata: crate::model::ObjectMetadata, + pub envelope: crate::model::PageEnvelope, + pub landed: crate::gpu::LandedCiphertext, +} + /// Internal transport result: native delivery stays materialized; HTTP relay /// delivery owns an unfinished connection and its exact opaque body framing. #[allow(clippy::large_enum_variant)] // Keep completed delivery inline without another allocation. pub enum RelayResponse { Complete(SignedResponse), + /// A page in a GPU landing buffer. Only a requester that asked for + /// landing gets this, and only that requester's sealed read uses it. + Landed(Box), + Http { authentication: crate::peer::forwarding::ForwardedHead, @@ -1074,6 +1208,7 @@ impl Transfers { receive_permit: Option>, failure: Rc>, mut timing: Option<&'a mut super::PageTiming<'_>>, + landing: Option, scope: &'a RequestScope, ) -> Operation<'a, RelayResponse> { Box::pin(async move { @@ -1159,7 +1294,7 @@ impl Transfers { // Native response verification requires an authenticated membership lease. let granting = now(); let native = if membership.is_some() { - self.accept_native(&request, plan, scope).await? + self.accept_native(&request, plan, landing, scope).await? } else { None }; @@ -1218,7 +1353,7 @@ impl Transfers { let (binding, accept, peer, pending) = native.ok_or(Error::Unauthorized)?; let arriving = now(); let received = self - .receive_native( + .receive_native_into( received.connection, authentication, binding, @@ -1230,8 +1365,7 @@ impl Transfers { length, scope, ) - .await - .map(RelayResponse::Complete); + .await; if received.is_ok() { self.record_rdma_stage(Event::RdmaArrivalCount, Event::RdmaArrivalNs, arriving); } From c6b7522f41266d9705a1996177ee14009f28a979 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:45:51 +0000 Subject: [PATCH 59/87] feat(racer): land gpu-read peer pages in GPU memory through the fill path Fill::acquire_sealed now tries land_from_peer after local disk staging: one CopyOnly request to the first provider with the request's GPU ordinal. A landed page becomes SealedSource::Landed and is decrypted on the GPU; it never joins single flight, memory, or disk. A host answer to the landing request is used once instead of fetching again. Any other outcome takes the usual path. Adds GPU helper tests for landing buffers and decrypting landed ciphertext. --- cmd/racer-dataplane/src/gpu.rs | 254 ++++++++++++++++++++- cmd/racer-dataplane/src/read/candidates.rs | 97 +++++++- cmd/racer-dataplane/src/read/fill.rs | 117 ++++++++++ 3 files changed, 457 insertions(+), 11 deletions(-) diff --git a/cmd/racer-dataplane/src/gpu.rs b/cmd/racer-dataplane/src/gpu.rs index 01fa26cfe..e2922c159 100644 --- a/cmd/racer-dataplane/src/gpu.rs +++ b/cmd/racer-dataplane/src/gpu.rs @@ -222,8 +222,11 @@ impl rdma_verbs::DmabufMemory for GpuLanding { /// Peer ciphertext that a NIC wrote into a GPU landing buffer. Holds the /// RDMA device page, so the slot is not reused until this is dropped. pub struct LandedCiphertext { - page: rdma_verbs::DevicePage, + /// Holds the slot. `None` only for test pages written without a NIC. + _page: Option, ordinal: i32, + offset: u64, + length: u64, } impl LandedCiphertext { @@ -234,11 +237,26 @@ impl LandedCiphertext { return Err(Error::Internal); } Ok(Self { - page, ordinal: landing.ordinal, + offset: page.offset(), + length: page.len() as u64, + _page: Some(page), }) } + /// Bytes at `offset` of `landing`, written there by a test in place of + /// a NIC. + #[cfg(test)] + pub(crate) fn for_test(landing: &GpuLanding, offset: u64, length: u64) -> Self { + assert!(offset + length <= landing.length); + Self { + _page: None, + ordinal: landing.ordinal, + offset, + length, + } + } + /// The GPU the bytes are on. pub fn ordinal(&self) -> i32 { self.ordinal @@ -246,17 +264,17 @@ impl LandedCiphertext { /// Bytes that landed. pub fn len(&self) -> u64 { - self.page.len() as u64 + self.length } /// Never true: landed ciphertext always holds a tag. pub fn is_empty(&self) -> bool { - self.page.is_empty() + self.length == 0 } /// Byte offset in the landing buffer. fn offset(&self) -> u64 { - self.page.offset() + self.offset } } @@ -1270,6 +1288,80 @@ pub(crate) mod testing { SIM.with(|sim| sim.borrow().as_ref().unwrap().0.clone()) } + /// A test scope with a few seconds on the real clock. + fn scope() -> RequestScope { + RequestScope::new( + crate::model::RequestId([1; 16]), + std::time::Instant::now() + std::time::Duration::from_secs(5), + ) + .unwrap() + } + + /// GPU `ordinal`'s landing buffer of `length` bytes, blocking. + pub(crate) fn landing( + helper: &GpuHelper, + ordinal: i32, + length: u64, + ) -> Result> { + futures::executor::block_on(helper.landing(ordinal, length, &scope())) + } + + /// Write `bytes` at `at` into a landing buffer through its dma-buf, as + /// a NIC would. + pub(crate) fn nic_write(landing: &GpuLanding, at: u64, bytes: &[u8]) { + use std::os::unix::fs::FileExt; + let file = std::fs::File::from(landing.fd.try_clone().unwrap()); + file.write_all_at(bytes, at).unwrap(); + } + + /// Import a fresh allocation on GPU `ordinal` as request `id`. + pub(crate) fn import(helper: &GpuHelper, id: u64, ordinal: i32) -> OwnedFd { + let size = racer_gpu::simulation::GRANULARITY; + let fd = helper.run(move || simulation().allocate(ordinal, size)); + let (reply, answer) = oneshot::channel(); + helper + .send(Job::Import { + id, + fd: fd.try_clone().unwrap(), + allocation_size: size, + window: 0..size, + reply, + }) + .unwrap(); + assert_eq!(futures::executor::block_on(answer).unwrap(), Ok(ordinal)); + fd + } + + /// Verify and decrypt `page`'s `range` into request `id` at offset 0. + pub(crate) fn write( + helper: &GpuHelper, + id: u64, + page: Box, + range: Range, + ) -> Result<()> { + let (reply, answer) = oneshot::channel(); + helper + .send(Job::Write { + id, + offset: 0, + work: Work::Sealed(Sealed { page, range }), + reply, + }) + .unwrap(); + futures::executor::block_on(answer).unwrap().1 + } + + /// Host-to-device copies the GPU has made so far. + pub(crate) fn host_copies(helper: &GpuHelper) -> usize { + helper.run(|| { + simulation() + .trace() + .iter() + .filter(|event| event.operation == racer_gpu::simulation::Operation::CopyH2d) + .count() + }) + } + /// A required-mode helper on simulated GPUs with these NUMA nodes. pub(crate) fn helper(numa: &[Option]) -> Arc { let devices: Vec = numa @@ -1294,3 +1386,155 @@ pub(crate) mod testing { .unwrap() } } + +#[cfg(test)] +mod tests { + //! RDMA landing buffers on simulated GPUs: allocation, reuse, and + //! decrypting ciphertext a NIC wrote straight into GPU memory. + use super::testing::*; + use super::*; + use crate::model::*; + use rdma_verbs::DmabufMemory; + + const LENGTH: u64 = 2 * SLOT_BYTES; + + fn metadata(length: u64) -> ObjectMetadata { + ObjectMetadata { + content_type: None, + version: ObjectVersion { + object: ObjectId { + cache: racer_control_wire::CacheId( + "00000000-0000-4000-8000-000000000001".into(), + ), + key: CacheKey([0; 32]), + }, + etag: StrongEtag::test_value("v1"), + }, + length, + expires_at: ExpiresAt::from_system_time(std::time::UNIX_EPOCH).unwrap(), + } + } + + /// A sealed page whose real ciphertext a NIC wrote at `at` in `landing`. + fn landed(landing: &GpuLanding, at: u64, plaintext: &[u8], tamper: bool) -> Box { + let metadata = metadata(plaintext.len() as u64); + let envelope = PageEnvelope { + cipher: PageCipher::Xaes256Gcm, + page: PageId { + version: metadata.version.clone(), + number: PageNumber(0), + }, + key_id: key_id_from_generation(1, 1).unwrap(), + nonce: Nonce([2; 24]), + plaintext_length: plaintext.len() as u32, + ciphertext_length: plaintext.len() as u32 + 16, + }; + let aad = crate::security::page_aad(&envelope).unwrap(); + let mut sealed = vec![0; plaintext.len() + 16]; + racer_crypto::seal(&[7; 32], &envelope.nonce.0, &aad, plaintext, &mut sealed).unwrap(); + if tamper { + sealed[3] ^= 1; + } + nic_write(landing, at, &sealed); + let page = Box::new(SealedPage { + metadata, + aad, + key: racer_crypto::xaes_derive(&[7; 32], &envelope.nonce.0), + expected_crc: None, + source: SealedSource::Landed(LandedCiphertext::for_test( + landing, + at, + sealed.len() as u64, + )), + envelope, + }); + page.validate_for(&page.envelope.page.clone()).unwrap(); + page + } + + #[test] + /// One landing buffer per GPU, made once and reused; each GPU gets its + /// own NIC address range; bad sizes and non-decrypting GPUs are refused. + fn landing_buffers_are_per_gpu_and_reused() { + let helper = helper(&[Some(0), Some(1)]); + let first = landing(&helper, 0, LENGTH).unwrap(); + assert!(Arc::ptr_eq(&first, &landing(&helper, 0, LENGTH).unwrap())); + let second = landing(&helper, 1, LENGTH).unwrap(); + assert_eq!((first.ordinal(), second.ordinal()), (0, 1)); + assert_eq!((first.offset(), first.length()), (0, LENGTH as usize)); + assert_ne!(first.iova(), second.iova()); + assert_eq!(landing(&helper, 2, LENGTH).err(), Some(Error::Unavailable)); + assert_eq!(landing(&helper, 0, 0).err(), Some(Error::Unavailable)); + assert_eq!(landing(&helper, 0, 4095).err(), Some(Error::Unavailable)); + assert_eq!(helper.run(|| simulation().live().staging), 2); + assert_eq!(helper.leaked(), 0); + } + + #[test] + /// Allocation or export failures are reported and can be retried. + fn landing_fails_when_allocation_or_export_fails() { + let helper = helper(&[None]); + helper.run(|| simulation().fault(racer_gpu::simulation::Operation::Staging, -4)); + assert_eq!(landing(&helper, 0, LENGTH).err(), Some(Error::Overloaded)); + helper.run(|| simulation().fault(racer_gpu::simulation::Operation::Dmabuf, -5)); + assert_eq!(landing(&helper, 0, LENGTH).err(), Some(Error::Unavailable)); + assert!(landing(&helper, 0, LENGTH).is_ok()); + } + + #[test] + /// Landed ciphertext decrypts on the GPU with no host-to-device copy of + /// the ciphertext, whole or in part. + fn landed_ciphertext_decrypts_on_the_gpu_without_a_host_copy() { + let helper = helper(&[None]); + let buffer = landing(&helper, 0, LENGTH).unwrap(); + let plaintext: Vec = (0..5000).map(|i| (i % 251) as u8).collect(); + let fd = import(&helper, 1, 0); + let before = host_copies(&helper); + write( + &helper, + 1, + landed(&buffer, SLOT_BYTES, &plaintext, false), + 0..5000, + ) + .unwrap(); + assert_eq!( + racer_gpu::simulation::read_allocation(&fd, 0, 5000), + plaintext + ); + assert_eq!(host_copies(&helper), before, "no host copy"); + assert_eq!(helper.counters().landed.load(Ordering::Relaxed), 1); + let fd = import(&helper, 2, 0); + write(&helper, 2, landed(&buffer, 0, &plaintext, false), 10..20).unwrap(); + assert_eq!( + racer_gpu::simulation::read_allocation(&fd, 0, 10), + plaintext[10..20] + ); + assert_eq!(helper.counters().landed.load(Ordering::Relaxed), 2); + assert_eq!(helper.leaked(), 0); + } + + #[test] + /// A tampered landed page is rejected before anything is written; a page + /// on another GPU's landing buffer is refused. + fn tampered_or_misplaced_landed_ciphertext_is_rejected() { + let helper = helper(&[None, None]); + let near = landing(&helper, 0, LENGTH).unwrap(); + let far = landing(&helper, 1, LENGTH).unwrap(); + let plaintext = vec![5; 100]; + let fd = import(&helper, 1, 0); + assert_eq!( + write(&helper, 1, landed(&near, 0, &plaintext, true), 0..100), + Err(Error::CorruptRecord) + ); + assert_eq!( + racer_gpu::simulation::read_allocation(&fd, 0, 100), + vec![0; 100] + ); + assert_eq!(helper.counters().rejected.load(Ordering::Relaxed), 1); + assert_eq!( + write(&helper, 1, landed(&far, 0, &plaintext, false), 0..100), + Err(Error::Internal) + ); + assert_eq!(helper.counters().landed.load(Ordering::Relaxed), 0); + } +} diff --git a/cmd/racer-dataplane/src/read/candidates.rs b/cmd/racer-dataplane/src/read/candidates.rs index c5c1c898c..8a9e696dc 100644 --- a/cmd/racer-dataplane/src/read/candidates.rs +++ b/cmd/racer-dataplane/src/read/candidates.rs @@ -9,6 +9,7 @@ use crate::model::AttemptId; use crate::model::MetadataSelector; use crate::model::ObjectId; use crate::model::PageNumber; +use crate::peer::Delivered; use crate::peer::Requester; use crate::peer::forwarding::VerifiedResponse; use crate::peer::protocol::FetchMode; @@ -1426,6 +1427,82 @@ impl CandidatePolicy { request_mode: RequestMode, trail: Option<(&mut CandidateTrail, u8)>, ) -> Result { + match self + .request_delivered( + membership, + destination, + context, + operation, + mode, + scope, + budget, + remaining_opportunities, + request_mode, + trail, + None, + ) + .await? + { + Delivered::Response(response) => Ok(response), + Delivered::Landed(_) => Err(Error::Internal), + } + } + + /// One CopyOnly page request for a `gpu-read` on GPU `ordinal`: the page + /// may land in a GPU landing buffer (see [`Requester::request_landing`]). + /// The caller sends it to one provider only. + #[allow(clippy::too_many_arguments)] // Same route inputs as a plain request. + pub(crate) async fn request_landing( + &self, + membership: &std::sync::Arc, + destination: &NodeId, + context: &OriginContext, + page: &crate::model::PageId, + scope: &RequestScope, + budget: &mut AcquisitionBudget, + ordinal: i32, + ) -> Result { + let operation = PeerOperation::Page { + page: page.clone(), + mode: FetchMode::CopyOnly, + }; + self.request_delivered( + membership, + destination, + context, + &operation, + FetchMode::CopyOnly, + scope, + budget, + 2, + RequestMode::SharedRoute, + None, + Some(ordinal), + ) + .await + } + + /// Whether this node can land peer pages on GPU `ordinal`. + pub(crate) fn landing_ready(&self, ordinal: i32) -> bool { + self.peers.landing_ready(ordinal) + } + + /// Request a peer copy, optionally into a GPU landing buffer. + #[allow(clippy::too_many_arguments)] // Keep the optional borrowed trace separate from route state. + async fn request_delivered( + &self, + membership: &std::sync::Arc, + destination: &NodeId, + context: &OriginContext, + operation: &PeerOperation, + mode: FetchMode, + scope: &RequestScope, + budget: &mut AcquisitionBudget, + remaining_opportunities: u32, + request_mode: RequestMode, + trail: Option<(&mut CandidateTrail, u8)>, + landing: Option, + ) -> Result { let mut remote = [b'?'; 36]; if trail.is_some() && racer_crypto::identity::canonical_uuid(&destination.0) { remote.copy_from_slice(destination.0.as_bytes()); @@ -1547,12 +1624,18 @@ impl CandidatePolicy { }, }; let registration = scope.cancellation.subscribe()?; - let mut exchange = if matches!(request_mode, RequestMode::DirectHedge) { + let mut exchange: Operation<'_, Delivered> = if let Some(ordinal) = landing { self.peers - .request_direct(request, membership.clone(), &attempt_scope) + .request_landing(request, membership.clone(), ordinal, &attempt_scope) } else { - self.peers - .request(request, membership.clone(), &attempt_scope) + let exchange = if matches!(request_mode, RequestMode::DirectHedge) { + self.peers + .request_direct(request, membership.clone(), &attempt_scope) + } else { + self.peers + .request(request, membership.clone(), &attempt_scope) + }; + Box::pin(async move { exchange.await.map(Delivered::Response) }) }; let mut stopped = None; facts.site = CandidateSite::Exchange; @@ -1573,7 +1656,8 @@ impl CandidatePolicy { .await; facts.raw = Some(match &response { Err(error) => CandidateOutcome::Error(*error), - Ok(response) => match response.response() { + Ok(Delivered::Landed(_)) => CandidateOutcome::Success, + Ok(Delivered::Response(response)) => match response.response() { PeerResponse::Miss => CandidateOutcome::Miss, PeerResponse::Unavailable => { CandidateOutcome::ResponseError(Error::Unavailable) @@ -1605,7 +1689,8 @@ impl CandidatePolicy { links: budget.remaining_links(), }), ), - Ok(response) => { + Ok(Delivered::Landed(_)) => {} + Ok(Delivered::Response(response)) => { let error = match response.response() { PeerResponse::Overloaded => Some(Error::Overloaded), PeerResponse::Unavailable => Some(Error::Unavailable), diff --git a/cmd/racer-dataplane/src/read/fill.rs b/cmd/racer-dataplane/src/read/fill.rs index 038224b32..e662928af 100644 --- a/cmd/racer-dataplane/src/read/fill.rs +++ b/cmd/racer-dataplane/src/read/fill.rs @@ -941,6 +941,16 @@ impl Fill { Err(_) => {} } } + match self + .land_from_peer(&page, membership.clone(), context, scope, budget, target) + .await + { + Ok(Some(fetched)) => return Ok(fetched), + Ok(None) => {} + Err(error @ (Error::Cancelled | Error::DeadlineExceeded)) => return Err(error), + // Any other failure takes the usual path through host memory. + Err(_) => {} + } let acquired = self .acquire_with_prefetch( page.clone(), @@ -967,6 +977,113 @@ impl Fill { } } + /// Fetch a page from a peer straight into a GPU landing buffer (GPUDirect + /// RDMA). This is a private fetch for one sealed read: it does not join + /// or create a shared single flight, and a landed page is never published + /// to memory or persisted, because there are no host bytes to share. + /// + /// Only the first provider in visit order is asked, with one CopyOnly + /// request. None means use the usual path: no landing buffer on the + /// request's GPU, a local copy exists, this node owns the page, or the + /// provider missed or failed. A page the provider sent to host memory + /// instead (no free landing slot, or HTTP) is used privately in the same + /// way, so it is not fetched twice. + async fn land_from_peer( + &self, + page: &PageId, + membership: Arc, + context: &OriginContext, + scope: &RequestScope, + budget: &mut AcquisitionBudget, + target: &super::sealed::SealedTarget, + ) -> Result> { + let resolver = &self.dependencies.candidates; + if !resolver.landing_ready(target.ordinal) + || self + .dependencies + .writer + .copy_only(page) + .ok() + .flatten() + .is_some() + { + return Ok(None); + } + if let Some(local) = self.acquire_local_copy(page, scope, false).await? { + return Ok(Some(match local { + AcquiredPage::Plaintext(result) => super::sealed::Fetched::Page(result), + AcquiredPage::Ciphertext(copy) => match self.seal(page, copy) { + Ok(sealed) => super::sealed::Fetched::Sealed(sealed), + Err(_) => return Ok(None), + }, + })); + } + let candidates = resolver + .candidates_scoped(membership.clone(), &page.version.object, page.number, scope) + .await?; + if resolver.is_candidate(&candidates) { + return Ok(None); + } + let order = resolver.visit_order(&candidates, &page.version.object, page.number); + let Some(first) = order.ranks().next() else { + return Ok(None); + }; + let destination = candidates.ordered[first].clone(); + let delivered = resolver + .request_landing( + &membership, + &destination, + context, + page, + scope, + budget, + target.ordinal, + ) + .await?; + let landed = match delivered { + crate::peer::Delivered::Landed(landed) => landed, + // The provider sent the page to host memory (no free landing + // slot, or HTTP): use it privately rather than fetch it again. + crate::peer::Delivered::Response(response) => { + let Ok(copy) = response_copy(response.response(), page) else { + return Ok(None); + }; + return Ok(self + .seal( + page, + UnverifiedPage { + copy, + disk_token: None, + }, + ) + .ok() + .map(super::sealed::Fetched::Sealed)); + } + }; + let crate::peer::transport::LandedResponse { + metadata, + envelope, + landed, + .. + } = *landed; + if landed.ordinal() != target.ordinal || envelope.page != *page { + return Ok(None); + } + metadata.immutable().validate_page(&envelope)?; + let sealed = Box::new(super::sealed::SealedPage { + aad: crate::security::page_aad(&envelope)?, + key: self.dependencies.crypto.derive_page_key(&envelope)?, + // No host bytes to checksum: the GPU checks the AEAD tag. + expected_crc: None, + metadata, + envelope, + source: super::sealed::SealedSource::Landed(landed), + }); + sealed.validate_for(page)?; + self.metrics.record(Event::PeerHit, 1); + Ok(Some(super::sealed::Fetched::Sealed(sealed))) + } + /// Package host ciphertext for GPU verification. fn seal(&self, page: &PageId, copy: UnverifiedPage) -> Result> { validate_copy(©.copy, page)?; From f2f51833b2013589268bec771c5a96fe491a948a Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:47:13 +0000 Subject: [PATCH 60/87] test(racer): cover gpu-read peer landing through the fill path --- cmd/racer-dataplane/src/peer.rs | 43 +++- .../src/read/tests/peer_copies.rs | 242 ++++++++++++++++++ 2 files changed, 284 insertions(+), 1 deletion(-) diff --git a/cmd/racer-dataplane/src/peer.rs b/cmd/racer-dataplane/src/peer.rs index 8e10fd944..ac4396236 100644 --- a/cmd/racer-dataplane/src/peer.rs +++ b/cmd/racer-dataplane/src/peer.rs @@ -495,8 +495,16 @@ pub enum Requester { bool, ) -> Operation<'static, VerifiedResponse>, >, + /// Test-only GPU landing: the landing GPU, and a script that returns + /// a landed page for a request or None to answer it as a plain request. + #[cfg(test)] + landing: Option<(i32, ScriptedLanding)>, }, } + +/// A scripted landed page for one request, or None. +#[cfg(test)] +pub(crate) type ScriptedLanding = Box Option>; impl Requester { #[cfg(any(test, feature = "subscription-interop"))] pub(crate) fn scripted( @@ -532,8 +540,26 @@ impl Requester { } }) }), + #[cfg(test)] + landing: None, }) } + + /// Test-only: let a scripted requester land pages on GPU `ordinal`. + #[cfg(test)] + pub(crate) fn with_scripted_landing( + self: Rc, + ordinal: i32, + land: ScriptedLanding, + ) -> Rc { + let mut requester = Rc::try_unwrap(self) + .ok() + .expect("unshared scripted requester"); + if let Self::Scripted { landing, .. } = &mut requester { + *landing = Some((ordinal, land)); + } + Rc::new(requester) + } pub fn new( paths: Rc, forwarding: Rc, @@ -746,8 +772,18 @@ impl Requester { } #[cfg(any(test, feature = "subscription-interop"))] Self::Scripted { - request: script, .. + request: script, + #[cfg(test)] + landing, + .. } => { + #[cfg(test)] + if let Some((own, land)) = landing + && *own == ordinal + && let Some(landed) = land(&request) + { + return Box::pin(async move { Ok(Delivered::Landed(Box::new(landed))) }); + } let response = script(request, membership, scope.clone(), false); Box::pin(async move { response.await.map(Delivered::Response) }) } @@ -758,6 +794,11 @@ impl Requester { pub(crate) fn landing_ready(&self, ordinal: i32) -> bool { match self { Self::Network { transfers, .. } => transfers.landing_ready(ordinal), + #[cfg(test)] + Self::Scripted { + landing: Some((own, _)), + .. + } => *own == ordinal, #[cfg(any(test, feature = "subscription-interop"))] Self::Scripted { .. } => false, } diff --git a/cmd/racer-dataplane/src/read/tests/peer_copies.rs b/cmd/racer-dataplane/src/read/tests/peer_copies.rs index 3dcce9a8b..4128c680c 100644 --- a/cmd/racer-dataplane/src/read/tests/peer_copies.rs +++ b/cmd/racer-dataplane/src/read/tests/peer_copies.rs @@ -2970,3 +2970,245 @@ fn registered_copy_out_refused_by_admission_keeps_the_registered_page() { crate::rdma::links::tests::pump(&mut w.a.owner, &mut w.b.owner); assert_eq!(free_buffers(&w), Some(2)); } + +/// GPU landing for `gpu-read` (GPUDirect RDMA) through `Fill::acquire_sealed`. +mod landing { + use super::*; + use crate::gpu::GpuHelper; + use crate::read::sealed::Fetched; + use crate::read::sealed::SealedSource; + use crate::read::sealed::SealedTarget; + + struct Landing { + f: Fixture, + peers: Rc, + ordered: Vec, + helper: Arc, + copy: CiphertextCopy, + /// Requests the landing script saw. + asked: Rc>, + } + + /// A non-owner whose first provider may land page 0 on GPU `ordinal`. + /// With `lands`, the script answers with the real ciphertext written into + /// the landing buffer, as a NIC would; otherwise the request goes to the + /// usual peer script. + fn landing(ordinal: i32, lands: bool) -> Landing { + let mut f = fixture(); + let copy = encrypted_copy(&mut f); + let (peers, ordered) = install_peers(&mut f, None); + let helper = crate::gpu::testing::helper(&[None, None]); + let buffer = crate::gpu::testing::landing(&helper, 0, 2 * crate::gpu::SLOT_BYTES).unwrap(); + crate::gpu::testing::nic_write(&buffer, 0, copy.ciphertext.bytes()); + let asked = Rc::new(Cell::new(0)); + let envelope = copy.ciphertext.envelope().clone(); + let metadata = copy.metadata.clone(); + let length = copy.ciphertext.bytes().len() as u64; + let seen = asked.clone(); + let land: crate::peer::ScriptedLanding = Box::new(move |request| { + seen.set(seen.get() + 1); + assert!(matches!( + request.operation, + PeerOperation::Page { + mode: FetchMode::CopyOnly, + .. + } + )); + if !lands { + return None; + } + Some(crate::peer::transport::LandedResponse { + // The fill path never reads the head: the transport verified it. + authentication: crate::peer::forwarding::ForwardedHead { + original: Arc::new(peer_wire::SignedHead { + head: http1::MessageHead { + start: http1::StartLine::Request { + method: "POST".into(), + target: "/test".into(), + }, + headers: vec![], + }, + signature: vec![], + }), + hops: vec![], + }, + metadata: metadata.clone(), + envelope: envelope.clone(), + landed: crate::gpu::LandedCiphertext::for_test(&buffer, 0, length), + }) + }); + let mut dependencies = f.fill.dependencies.clone(); + dependencies.candidates = Rc::new(CandidatePolicy::new( + node(peers.local), + Rc::new(Placement::new(16)), + Requester::scripted( + peers.clone(), + ScriptedPeers::direct_hedge_available, + ScriptedPeers::request, + ScriptedPeers::request_direct, + ) + .with_scripted_landing(ordinal, land), + dependencies.credentials.clone(), + Arc::new(controlplane::Published::new( + crate::control::Snapshot::retention(2), + )), + )); + f.fill = Fill::new(dependencies); + f.fill + .dependencies + .writer + .retention() + .set_ownership(Rc::new(|_| false)); + Landing { + f, + peers, + ordered, + helper, + copy, + asked, + } + } + + fn acquire_sealed(l: &mut Landing) -> Result { + let target = SealedTarget { + helper: l.helper.clone(), + ordinal: 0, + }; + let mut budget = AcquisitionBudget::new(l.f.scope.deadline.0, 16, 32); + let f = &mut l.f; + drive( + f.fill.acquire_sealed( + f.page.clone(), + f.membership.clone(), + &f.context, + &f.scope, + &mut budget, + &target, + ), + &mut f.engine, + &f.crypto, + ) + } + + fn landed_count(helper: &GpuHelper) -> u64 { + helper + .counters() + .landed + .load(std::sync::atomic::Ordering::Relaxed) + } + + #[test] + /// A landed page decrypts on the GPU and is private to this read: no + /// CPU decrypt, no memory publication, no persistence, no host + /// ciphertext charge, and a later plain read fetches it again. + fn landed_peer_page_decrypts_on_the_gpu_and_stays_private() { + let queue = Rc::new(uring_runtime::drivers::DriverQueue::new(1024)); + let _owner = queue.enter(); + let mut l = landing(0, true); + let ciphertext = + l.f.fill + .dependencies + .admission + .used(ResourceClass::Ciphertext); + let Fetched::Sealed(sealed) = acquire_sealed(&mut l).unwrap() else { + panic!("expected a sealed page"); + }; + assert!(matches!(sealed.source, SealedSource::Landed(_))); + assert!(sealed.expected_crc.is_none()); + // One landing request to the first provider, none through the script. + assert_eq!(l.asked.get(), 1); + assert!(l.peers.calls.borrow().is_empty()); + assert_unpublished(&l.f); + assert_eq!( + l.f.fill + .dependencies + .admission + .used(ResourceClass::Ciphertext), + ciphertext + ); + let report = sealed.report(); + assert!(report.copy.is_none() && report.token.is_none()); + let fd = crate::gpu::testing::import(&l.helper, 1, 0); + crate::gpu::testing::write(&l.helper, 1, sealed, 0..3).unwrap(); + assert_eq!(racer_gpu::simulation::read_allocation(&fd, 0, 3), b"abc"); + assert_eq!(landed_count(&l.helper), 1); + // A passed verdict persists nothing: there is no host copy. + l.f.fill.sealed_verdict(report, true); + assert_unpublished(&l.f); + // Nothing was shared: a later plain read fetches the page again. + let deadline = l.f.scope.deadline.0; + l.peers + .replies + .borrow_mut() + .push_back((l.ordered[0].clone(), Reply::Copy(l.copy.clone()))); + let result = acquire(&mut l.f, &mut AcquisitionBudget::new(deadline, 16, 32)).unwrap(); + assert_eq!(result.plaintext.bytes(), b"abc"); + assert_eq!(l.peers.calls.borrow().len(), 1); + drop(fd); + assert_eq!(l.helper.leaked(), 0); + } + + #[test] + /// No free landing slot at this node: the provider sends the page to + /// host memory, and that answer is used once rather than fetched again. + fn host_answer_to_a_landing_request_is_used_once_and_not_published() { + let queue = Rc::new(uring_runtime::drivers::DriverQueue::new(1024)); + let _owner = queue.enter(); + let mut l = landing(0, false); + l.peers + .replies + .borrow_mut() + .push_back((l.ordered[0].clone(), Reply::Copy(l.copy.clone()))); + let Fetched::Sealed(sealed) = acquire_sealed(&mut l).unwrap() else { + panic!("expected a sealed page"); + }; + assert!(matches!(sealed.source, SealedSource::Host(_))); + assert_eq!(l.asked.get(), 1); + assert_eq!(l.peers.calls.borrow().len(), 1, "not fetched twice"); + assert!( + l.f.fill + .dependencies + .memory + .get(&l.f.page) + .unwrap() + .is_none() + ); + let fd = crate::gpu::testing::import(&l.helper, 1, 0); + crate::gpu::testing::write(&l.helper, 1, sealed, 0..3).unwrap(); + assert_eq!(racer_gpu::simulation::read_allocation(&fd, 0, 3), b"abc"); + assert_eq!(landed_count(&l.helper), 0); + } + + #[test] + /// A landing buffer on another GPU is never used, and a failed landing + /// request falls back to the usual acquisition. + fn landing_falls_back_to_the_usual_path() { + let queue = Rc::new(uring_runtime::drivers::DriverQueue::new(1024)); + let _owner = queue.enter(); + let mut l = landing(1, true); + l.peers + .replies + .borrow_mut() + .push_back((l.ordered[0].clone(), Reply::Copy(l.copy.clone()))); + let fetched = acquire_sealed(&mut l).unwrap(); + assert!(!matches!( + fetched, + Fetched::Sealed(ref sealed) if matches!(sealed.source, SealedSource::Landed(_)) + )); + assert_eq!(l.asked.get(), 0); + assert_eq!(l.peers.calls.borrow().len(), 1); + let mut l = landing(0, false); + { + let mut replies = l.peers.replies.borrow_mut(); + replies.push_back((l.ordered[0].clone(), Reply::Unauthorized)); + for node in &l.ordered { + replies.push_back((node.clone(), Reply::Copy(l.copy.clone()))); + } + } + let fetched = acquire_sealed(&mut l); + assert!(fetched.is_ok(), "{:?}", fetched.err()); + assert_eq!(l.asked.get(), 1); + assert!(l.peers.calls.borrow().len() >= 2); + assert_eq!(landed_count(&l.helper), 0); + } +} From 501a2b74b577c5c564da62228c50614442f9f825 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:48:17 +0000 Subject: [PATCH 61/87] test(racer): land a native peer page in GPU memory over simulated RDMA --- cmd/racer-dataplane/src/gpu.rs | 2 +- cmd/racer-dataplane/src/peer/transport.rs | 112 +++++++++++++++++--- cmd/racer-dataplane/src/rdma/links/tests.rs | 2 +- 3 files changed, 98 insertions(+), 18 deletions(-) diff --git a/cmd/racer-dataplane/src/gpu.rs b/cmd/racer-dataplane/src/gpu.rs index e2922c159..31eeb1c30 100644 --- a/cmd/racer-dataplane/src/gpu.rs +++ b/cmd/racer-dataplane/src/gpu.rs @@ -273,7 +273,7 @@ impl LandedCiphertext { } /// Byte offset in the landing buffer. - fn offset(&self) -> u64 { + pub(crate) fn offset(&self) -> u64 { self.offset } } diff --git a/cmd/racer-dataplane/src/peer/transport.rs b/cmd/racer-dataplane/src/peer/transport.rs index 19abee990..0503ee4c8 100644 --- a/cmd/racer-dataplane/src/peer/transport.rs +++ b/cmd/racer-dataplane/src/peer/transport.rs @@ -1968,6 +1968,8 @@ mod tests { receiver_site: Option<&'static str>, /// The sender's link breaks before it writes. sender_down: bool, + /// The receiver grants a slot of a simulated GPU's landing buffer. + landing: bool, } struct PageOutcome { @@ -1975,6 +1977,8 @@ mod tests { result: Result>, /// The received ciphertext was still in registered memory. registered: bool, + /// The page landed in the GPU landing buffer. + landed: bool, /// Free buffers while the response was still held. held: (Option, Option), sent: Result, @@ -2141,17 +2145,27 @@ mod tests { }; let sender_membership = page_membership(&signers, case.sender_site); let receiver_membership = page_membership(&signers, case.receiver_site); + let gpu = case.landing.then(|| { + let helper = crate::gpu::testing::helper(&[None]); + let bytes = (crate::rdma::BUFFER_LEN as u64) + .next_multiple_of(crate::config::GPU_STAGING_UNIT); + crate::rdma::links::tests::attach_landing(&mut w, &helper, bytes); + assert_eq!(w.a.links.landing_ordinal(RailId(7)), Some(0)); + helper + }); // The receiver grants in-band and names the grant in its signed Accept. - let pending = crate::rdma::links::tests::drive( - &mut w.a.owner, - &mut w.b.owner, - receiver.links.as_ref().unwrap().grant( - signers[2].node(), - RailId(7), - binding.transfer, - &scope, - ), - ) + let links = receiver.links.as_ref().unwrap(); + let pending = crate::rdma::links::tests::drive(&mut w.a.owner, &mut w.b.owner, async { + if case.landing { + links + .grant_landing(signers[2].node(), RailId(7), binding.transfer, 0, &scope) + .await + } else { + links + .grant(signers[2].node(), RailId(7), binding.transfer, &scope) + .await + } + }) .unwrap(); let accept = binding .sign( @@ -2207,10 +2221,11 @@ mod tests { return receiver .wire .1 - .response(auth, body.bytes()?.to_vec(), &scope); + .response(auth, body.bytes()?.to_vec(), &scope) + .map(RelayResponse::Complete); }; receiver - .receive_native( + .receive_native_into( received.connection, auth, binding.clone(), @@ -2291,17 +2306,40 @@ mod tests { let held = free(&w); let registered = matches!( &result, - Ok(SignedResponse { response: PeerResponse::Page { ciphertext, .. }, .. }) - if ciphertext.is_registered() + Ok(RelayResponse::Complete(SignedResponse { + response: PeerResponse::Page { ciphertext, .. }, + .. + })) if ciphertext.is_registered() ); - let result = result.map(|response| match &response.response { - PeerResponse::Page { ciphertext, .. } => ciphertext.bytes().to_vec(), + let landed = matches!(&result, Ok(RelayResponse::Landed(_))); + let result = result.map(|response| match response { + RelayResponse::Complete(SignedResponse { + response: PeerResponse::Page { ciphertext, .. }, + .. + }) => ciphertext.bytes().to_vec(), + RelayResponse::Landed(landed) => { + // Read what the NIC wrote, through the landing dma-buf. + use std::os::unix::fs::FileExt; + assert_eq!(landed.envelope.ciphertext_length, 144); + let buffer = w.a.links.node().landing(RailId(7)).unwrap(); + let file = std::fs::File::from( + rdma_verbs::DmabufMemory::fd(&*buffer) + .try_clone_to_owned() + .unwrap(), + ); + let mut bytes = vec![0; landed.landed.len() as usize]; + file.read_exact_at(&mut bytes, landed.landed.offset()) + .unwrap(); + bytes + } _ => panic!("expected page"), }); + drop(gpu); crate::rdma::links::tests::pump(&mut w.a.owner, &mut w.b.owner); PageOutcome { result, registered, + landed, held, sent, completed: sender.native_completed.get(), @@ -2342,6 +2380,48 @@ mod tests { } } + #[test] + /// A landing grant over a ready link: the sender writes into the + /// receiver's GPU landing buffer, the receiver checks the signed head + /// for exactly this page and returns it as landed, and no host buffer + /// is used on the receiver. + fn simulated_landing_grant_lands_the_page_in_gpu_memory() { + for reverse in [false, true] { + let outcome = page_roundtrip(PageCase { + reverse, + landing: true, + ..PageCase::default() + }); + assert_eq!(page_bytes(&outcome.result), vec![0x5a; 144]); + assert!(outcome.landed && !outcome.registered); + assert_eq!(outcome.sent, Ok(true)); + assert_eq!( + (outcome.completed, outcome.fallbacks, outcome.completions), + (1, 0, 1) + ); + // No host receive buffer was used. + assert_eq!(outcome.held, (Some(2), Some(2))); + assert_eq!(outcome.free, (Some(2), Some(2))); + } + } + + #[test] + /// The sender may still answer a landing grant over HTTP (here its + /// link is down): the page arrives in host memory as usual. + fn simulated_landing_grant_falls_back_to_http() { + let outcome = page_roundtrip(PageCase { + landing: true, + sender_down: true, + ..PageCase::default() + }); + assert_eq!(page_bytes(&outcome.result), vec![0x5a; 144]); + assert!(!outcome.landed); + assert_eq!( + (outcome.completed, outcome.fallbacks, outcome.completions), + (0, 1, 0) + ); + } + #[test] fn simulated_sender_rejects_cross_site_and_releases_the_grant() { for site in ["site2", ""] { diff --git a/cmd/racer-dataplane/src/rdma/links/tests.rs b/cmd/racer-dataplane/src/rdma/links/tests.rs index f37866fe6..cc6a1ccf8 100644 --- a/cmd/racer-dataplane/src/rdma/links/tests.rs +++ b/cmd/racer-dataplane/src/rdma/links/tests.rs @@ -871,7 +871,7 @@ fn assign_landing_prefers_numa_then_any() { } /// Register a simulated GPU's landing buffer on `a`'s rail. -fn attach_landing(w: &mut World, helper: &Arc, bytes: u64) { +pub(crate) fn attach_landing(w: &mut World, helper: &Arc, bytes: u64) { w.a.links.node().set_gpu_landing(helper.clone(), bytes); let rails = w.a.links.active(); let scope = scope(); From 7d8d677a4873be4d15ad2024b8fedce85a0826a4 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:49:09 +0000 Subject: [PATCH 62/87] docs(racer): describe GPUDirect RDMA landing and rdma-verbs device pools --- deploy/racer/dataplane-config.yaml.tmpl | 4 ++ designs/racer-gpudirect.md | 62 +++++++++++++++++++++++-- designs/racer-verbs.md | 32 ++++++++++++- docs/content/reference/racer.md | 15 ++++++ 4 files changed, 108 insertions(+), 5 deletions(-) diff --git a/deploy/racer/dataplane-config.yaml.tmpl b/deploy/racer/dataplane-config.yaml.tmpl index 6a90f0cfe..110e25398 100644 --- a/deploy/racer/dataplane-config.yaml.tmpl +++ b/deploy/racer/dataplane-config.yaml.tmpl @@ -19,6 +19,10 @@ data: # Cap on ciphertext staging memory per GPU, in bytes: a nonzero multiple of # 2 MiB, default 256 MiB. Reserved for GPU decryption; not used yet. # RACER_GPU_STAGING_BYTES: "268435456" + # GPU memory per GPU that peers' NICs write GPU-read pages into over RDMA + # (GPUDirect RDMA), in bytes: a multiple of 2 MiB, default 256 MiB. 0 turns + # it off. Used only when both GPU reads and RDMA are on. + # RACER_GPU_LANDING_BYTES: "268435456" # No thread cap by default: eligible physical cores, CPU quota, and resource budgets apply. # I/O shards share NUMA-local crypto threads at approximately 2:1, using leftovers: # eight eligible cores on one NUMA node give five I/O plus three crypto threads diff --git a/designs/racer-gpudirect.md b/designs/racer-gpudirect.md index f43e521dc..d263ea9c6 100644 --- a/designs/racer-gpudirect.md +++ b/designs/racer-gpudirect.md @@ -191,7 +191,7 @@ Ciphertext reaches GPU staging memory owned by the dataplane in one of three way - **Memory, origin or HTTP peer:** one host-to-GPU copy. - **Local disk:** cuFile reads it straight from NVMe (aligned records only, below). -- **RDMA peer:** the NIC writes it into GPU memory registered with +- **RDMA peer:** the NIC writes it into a GPU landing buffer registered with `ibv_reg_dmabuf_mr`. The dma-buf comes from `cuMemGetHandleForAddressRange`. `RACER_GPU_STAGING_BYTES` caps the dataplane's ciphertext staging memory per GPU. @@ -232,8 +232,62 @@ above: and the page's owner drops that copy from memory and disk, as it does after a failed CPU decrypt. When it accepts a page that came from a peer, the owner may persist it, as it does after a CPU decrypt. -- RDMA peers do not yet write into GPU staging: an RDMA page lands in host - memory and takes one host-to-GPU copy. +- RDMA peers write `gpu-read` pages straight into GPU memory (GPUDirect RDMA + landing), described below. Other RDMA pages land in host memory and take + one host-to-GPU copy when a GPU read needs them. + +### GPUDirect RDMA landing + +When GPU decryption and RDMA are both on, each GPU can get one landing +buffer: GPU memory separate from the staging buffer, sized by +`RACER_GPU_LANDING_BYTES` (default 256 MiB per GPU, 0 turns landing off). +The GPU helper allocates it once, exports it once as a dma-buf, and keeps it +until the process exits. At each RDMA activation, every rail is matched with +at most one GPU, and every GPU with at most one rail: a GPU on the rail's +NUMA node first, then any free GPU (`assign_landing` in +`src/rdma/links.rs`). PCI distance is not used; only NUMA nodes are known on +both sides. The rail registers the landing buffer as its `rdma_verbs` device +pool, so the buffer is split into page-sized slots. If allocation, export, or +registration fails, the rail stays host-only. + +A `gpu-read` page then takes this path (`Fill::land_from_peer`): + +1. If this node is not a candidate for the page, has no local copy, and has + a landing buffer on the request's GPU, it sends one CopyOnly request to the + first provider in visit order, carrying the GPU ordinal. +2. When the hop is RDMA and the rail's landing buffer is on that GPU, the + receiver grants a landing slot instead of a host buffer. If no slot is + free, it grants host memory as usual. +3. The sender writes the ciphertext into the slot. After the usual fence, the + receiver checks the signed completion and that the signed response head + describes exactly this page (`landed_response_head`), and the requester + verifies the signature chain against its request binding. +4. The page becomes a sealed page whose ciphertext is in the landing slot. + The GPU checks the tag (there is no CRC: no host bytes were seen) and + decrypts into the client's allocation. + +A landed page is used once. It does not join or create a single flight, is +not published to memory, and is not persisted, because there are no host +bytes to share. The slot is held until the GPU job finishes. If the provider +answers with a host page (no free slot, or HTTP), that page is used once in +the same way, so it is not fetched twice. A miss, any failure, or a landing +buffer on another GPU falls back to the usual path. Host-only nodes and +non-GPU reads do not change. Relays never ask for landing. + +`racer_rdma_pages_landed_total` counts pages that landed, and the GPU +helper's `landed` counter counts landed pages it decrypted. A landed page the +GPU rejects fails the read like any other sealed page; there is no host copy +to drop. + +The GPU memory used for landing is not charged to the host admission +budget. It is bounded by `RACER_GPU_LANDING_BYTES` per GPU. + +Not covered yet: there is no end-to-end `gpu-read` test across two whole +dataplanes, because the test harness has no two-node fixture with simulated +RDMA and a simulated GPU on the client socket. Tests cover each layer +instead: verbs device pools, landing grants over two simulated nodes, the +peer transport over a simulated link, the fill path with a scripted +provider, and GPU decryption from a landing buffer. ### Disk layout v2 @@ -305,4 +359,4 @@ rule (`internal/racerobject/racerobject.go`). 1. Descriptor passing, `host-read`, the Python client and the vLLM plugin. 2. GPU discovery, the `gpu` crate, GPU decryption and `gpu-read`. 3. Aligned disk records and cuFile reads. -4. GPUDirect RDMA from peers. +4. GPUDirect RDMA from peers (landing for `gpu-read`, described above). diff --git a/designs/racer-verbs.md b/designs/racer-verbs.md index 786148efb..75c06e963 100644 --- a/designs/racer-verbs.md +++ b/designs/racer-verbs.md @@ -112,6 +112,35 @@ thread and returns the buffer to the pool. Dropping a pool with held, quarantined, or leaked buffers keeps the region and charge alive rather than deregistering memory prematurely. +## Device page pools + +A rail can also hold one device pool: caller memory that the NIC reaches +through a Linux dma-buf, such as GPU memory. The caller passes an owner that +implements `DmabufMemory` (descriptor, offset, length, and the NIC address of +the first byte) to `Client::attach_device`, once per rail per round, after +activation. The owner thread registers it with `ibv_reg_dmabuf_mr` +(`LOCAL_WRITE | MW_BIND`) and splits it into buffers of the node's buffer +length. The crate does not depend on any GPU library. + +`Client::grant_device` grants a device buffer. It uses the same states, the +same window bind, and the same fence as a host grant: the arrival resolves +only after the peer's write completion and a successful `LOCAL_INV`, so the +peer can no longer write the buffer. The arrival is `Arrival::Device` with a +`DevicePage`, which names the memory owner, the offset, and the length, and +holds the buffer until it is dropped. A device grant fails with `Unavailable` +when the rail has no device pool and `Overloaded` when no device buffer is +free; it never falls back to host memory. A window stays tied to the region +it first binds, so a link makes device windows on first use and keeps them +apart from host windows, at most one per concurrent grant. + +The CPU never maps or copies device memory. `DevicePage` has no byte API, the +region reports a null pointer, and sends, writes, and receives that name a +device region are rejected in Rust, in the C adapter, and in the simulation. +If deregistration fails, the region leaks with its memory owner and the node +guard, so the memory is never freed while the NIC might write it. The +simulation maps a memfd in place of the dma-buf (`MemfdMemory`), and traces +and faults the registration as `Operation::RegisterDmabuf`. + ## Links For a configured local grant limit `g`: @@ -188,7 +217,8 @@ this crate, remains the authoritative page outcome. There is no in-band ## Native ABI and completion checks -The adapter ABI is version 4. Rust requires that version and the current +The adapter ABI is version 5. Version 5 added `rdma_verbs_register_dmabuf` +and the rule that send, write, and receive posts reject dma-buf regions. Rust requires that version and the current symbols; there is no legacy ABI loader. The completion record is 32 bytes: ```c diff --git a/docs/content/reference/racer.md b/docs/content/reference/racer.md index f8c1e9612..57a3ff8e6 100644 --- a/docs/content/reference/racer.md +++ b/docs/content/reference/racer.md @@ -204,6 +204,7 @@ Numeric values are unsigned decimal strings: `268435456`, not `256Mi`. | `RACER_ENABLE_RDMA` | `auto` | `auto`, `true`, or `false`; see [modes](#rdma) | | `RACER_ENABLE_GPU` | `auto` | `auto`, `true`, or `false`; see [GPU reads](#gpu-reads) | | `RACER_GPU_STAGING_BYTES` | `268435456` (256 MiB) | Cap on ciphertext staging memory per GPU, in bytes. Must be a nonzero multiple of 2 MiB. Below about 34 MiB the GPU does not decrypt and the CPU does; see [GPU reads](#gpu-reads) | +| `RACER_GPU_LANDING_BYTES` | `268435456` (256 MiB) | GPU memory per GPU that peers' NICs write GPU-read pages into over RDMA, in bytes. A multiple of 2 MiB; `0` turns it off. Below one page slot (16 MiB plus 4 KiB) it is off; see [GPU reads](#gpu-reads) | | `RACER_SHARES` | `4` | Enrollment proposal; explicit Node `shares` wins | | `RACER_PLAINTEXT_BYTES` / `RACER_CIPHERTEXT_BYTES` | Unset: `67108864` (64 MiB) / `134217728` (128 MiB) per final I/O worker | Independent plaintext/ciphertext admission budgets; an explicit value is node-wide and divided among I/O workers | | `RACER_DIRTY_BYTES` | Unset: `167772160` (160 MiB) per final I/O worker | Dirty-data budget; an explicit value is node-wide | @@ -1036,9 +1037,23 @@ Where the bytes come from decides who decrypts them: - A page on local disk is read straight from the disk into GPU memory with cuFile when it can be. Otherwise the CPU reads it and copies it to the GPU. +- With RDMA on, a page from a peer can be written by the peer's NIC straight + into GPU memory on this node (GPUDirect RDMA). See below. + If the GPU finds a damaged page, the read fails and the dataplane drops that copy from memory or disk, as it does when the CPU finds one. +When both GPU reads and RDMA are on, each GPU can get a landing buffer of +`RACER_GPU_LANDING_BYTES`, split into page slots. Each RDMA rail gets at most +one GPU's landing buffer, preferring a GPU on the rail's NUMA node. A GPU +read that needs a page from a peer over that rail, on that GPU, asks the peer +to write the encrypted page into a free slot. The GPU then checks and +decrypts it. Such a page is used only by that read: it is not kept in memory, +written to disk, or shared with other reads. When no slot is free, the GPUs +do not match, or anything fails, the page comes through host memory as +before. The dataplane logs `stage=gpu-landing` lines at RDMA activation, and +`racer_rdma_pages_landed_total` counts pages that landed in GPU memory. + `RACER_GPU_STAGING_BYTES` is the GPU memory the dataplane keeps on each GPU for encrypted pages. It is split into slots of one page (16 MiB plus 4 KiB). Two slots are needed for decryption, so a smaller cap means the CPU decrypts. The From e1fdda58eb2726730bedc952802fc2d5e49a6a9b Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:52:58 +0000 Subject: [PATCH 63/87] chore(racer): satisfy clippy for GPU landing receive --- cmd/racer-dataplane/src/peer.rs | 1 + cmd/racer-dataplane/src/peer/transport.rs | 51 ++++------------------- 2 files changed, 9 insertions(+), 43 deletions(-) diff --git a/cmd/racer-dataplane/src/peer.rs b/cmd/racer-dataplane/src/peer.rs index ac4396236..0464c3170 100644 --- a/cmd/racer-dataplane/src/peer.rs +++ b/cmd/racer-dataplane/src/peer.rs @@ -911,6 +911,7 @@ impl Requester { ) -> Operation<'a, transport::RelayResponse> { self.exchange_inner_mode(request, membership, relay, scope, false, None, None) } + #[allow(clippy::too_many_arguments)] // Keep transport, routing, and landing inputs explicit. fn exchange_inner_mode<'a>( &'a self, request: SignedRequest, diff --git a/cmd/racer-dataplane/src/peer/transport.rs b/cmd/racer-dataplane/src/peer/transport.rs index 0503ee4c8..a23af0ce4 100644 --- a/cmd/racer-dataplane/src/peer/transport.rs +++ b/cmd/racer-dataplane/src/peer/transport.rs @@ -560,49 +560,14 @@ impl Transfers { let read = self.io.read_body_exact(connection, buffer, scope).await?; Ok((read.lease, read.buffer)) } - /// Receiver: read the sender's one signed outcome. `Complete` means the page - /// is in the granted window; anything after that is `Io`, because the HTTP - /// body was never sent. `Finish` carries the HTTP body. - #[cfg(test)] + /// Receiver: read the sender's one signed outcome. `Complete` means the + /// page is in the granted window; anything after that is `Io`, because + /// the HTTP body was never sent. `Finish` carries the HTTP body. A landing + /// grant ([`Self::accept_native`] with a GPU) whose page the sender wrote + /// comes back as [`RelayResponse::Landed`]; any other page as + /// [`RelayResponse::Complete`]. #[allow(clippy::too_many_arguments)] // Preserve explicit authenticated transfer inputs. pub(super) async fn receive_native( - &self, - connection: ConnectionLease, - authentication: ForwardedHead, - binding: Binding, - accept: SignedHead, - peer: NodeId, - control: SignedHead, - pending: PendingReceive, - membership: &std::sync::Arc, - length: usize, - scope: &RequestScope, - ) -> Result { - match self - .receive_native_into( - connection, - authentication, - binding, - accept, - peer, - control, - pending, - membership, - length, - scope, - ) - .await? - { - RelayResponse::Complete(response) => Ok(response), - _ => Err(Error::Internal), - } - } - - /// Like [`Self::receive_native`], but a landing grant - /// ([`Self::accept_native`] with a GPU) whose page the sender wrote - /// comes back as [`RelayResponse::Landed`]. - #[allow(clippy::too_many_arguments)] // Preserve explicit authenticated transfer inputs. - pub(super) async fn receive_native_into( &self, connection: ConnectionLease, authentication: ForwardedHead, @@ -1353,7 +1318,7 @@ impl Transfers { let (binding, accept, peer, pending) = native.ok_or(Error::Unauthorized)?; let arriving = now(); let received = self - .receive_native_into( + .receive_native( received.connection, authentication, binding, @@ -2225,7 +2190,7 @@ mod tests { .map(RelayResponse::Complete); }; receiver - .receive_native_into( + .receive_native( received.connection, auth, binding.clone(), From d966c26af9c9bdd821ad530f7a386a7a82be7199 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:57:45 +0000 Subject: [PATCH 64/87] fix(racer): settle failed landed GPU verdicts on the reading worker A page landed by RDMA has no host copy or disk mapping for its owner to drop, so routing its failed verdict to the owner only consumed a retained verdict slot. Record the corrupt miss locally instead. --- cmd/racer-dataplane/src/read/dispatch.rs | 47 +++++++++++++++++++++++- cmd/racer-dataplane/src/read/sealed.rs | 8 ++++ cmd/racer-dataplane/src/store.rs | 14 +++++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/cmd/racer-dataplane/src/read/dispatch.rs b/cmd/racer-dataplane/src/read/dispatch.rs index 27dac7366..d6d25c506 100644 --- a/cmd/racer-dataplane/src/read/dispatch.rs +++ b/cmd/racer-dataplane/src/read/dispatch.rs @@ -958,7 +958,9 @@ impl WorkerDirectory { let Ok(owner) = self.page_owner(&report.page) else { return; }; - if self.is_local(owner) { + // A landed page has no host copy or disk mapping for its owner to act + // on, so its verdict is settled here and never takes a retained slot. + if self.is_local(owner) || !report.has_owner_state() { if let Ok(local) = self.local() { local.fill.sealed_verdict(report, passed); } @@ -1627,6 +1629,14 @@ mod tests { .unwrap() } fn failed_report(page: PageId) -> super::super::sealed::SealedReport { + super::super::sealed::SealedReport { + page: page.clone(), + copy: None, + token: Some(crate::store::ReadToken::for_test(page)), + } + } + /// A landed page's report: nothing for the owner to persist or drop. + fn landed_report(page: PageId) -> super::super::sealed::SealedReport { super::super::sealed::SealedReport { page, copy: None, @@ -1702,6 +1712,41 @@ mod tests { assert_eq!(directory.mailboxes[1].take_failed_verdicts().len(), 1); } #[test] + fn failed_landed_verdict_settles_locally() { + let queue = Rc::new(uring_runtime::drivers::DriverQueue::new(64)); + let _queue = queue.enter(); + let workers = vec![WorkerId(0), WorkerId(1)]; + let directory = Arc::new( + WorkerDirectory::new( + Arc::new(WorkerMap::new(workers.clone()).unwrap()), + workers, + 4, + ) + .unwrap(), + ); + directory.mailboxes[1].install().unwrap(); + let page = page_owned_by(&directory, WorkerId(1)); + let (local, _fixture) = crate::read::tests::admission_coordinator(); + let before = local + .fill + .metrics + .count(crate::telemetry::Event::CorruptMiss); + { + let _local = directory.simulation_scope(Some((WorkerId(0), local.clone()))); + directory.report_sealed(landed_report(page), false, &scope()); + } + assert_eq!( + local + .fill + .metrics + .count(crate::telemetry::Event::CorruptMiss) + - before, + 1 + ); + assert_eq!(directory.mailboxes[1].outstanding(), 0); + assert!(directory.mailboxes[1].take_failed_verdicts().is_empty()); + } + #[test] fn failed_sealed_verdicts_are_bounded() { let directory = directory(1); let page = page_owned_by(&directory, WorkerId(1)); diff --git a/cmd/racer-dataplane/src/read/sealed.rs b/cmd/racer-dataplane/src/read/sealed.rs index ba9e40c2d..6943d078f 100644 --- a/cmd/racer-dataplane/src/read/sealed.rs +++ b/cmd/racer-dataplane/src/read/sealed.rs @@ -103,6 +103,14 @@ pub struct SealedReport { pub(crate) token: Option, } +impl SealedReport { + /// Whether the owner holds anything to persist or invalidate for this + /// page. Landed pages are used once and leave nothing behind. + pub(crate) fn has_owner_state(&self) -> bool { + self.copy.is_some() || self.token.is_some() + } +} + impl SealedPage { /// Owner-side facts about where the bytes came from. Cheap: it shares /// the ciphertext and copies no secrets. diff --git a/cmd/racer-dataplane/src/store.rs b/cmd/racer-dataplane/src/store.rs index 7717fd148..36c875cfa 100644 --- a/cmd/racer-dataplane/src/store.rs +++ b/cmd/racer-dataplane/src/store.rs @@ -101,6 +101,20 @@ pub struct ReadToken { page: PageId, location: RecordLocation, } +#[cfg(test)] +impl ReadToken { + /// A token for a mapping that need not exist, for tests that only route it. + pub(crate) fn for_test(page: PageId) -> Self { + Self { + page, + location: RecordLocation { + segment: page_alloc::SegmentId(0), + generation: page_alloc::Generation(1), + extent: page_alloc::Extent::new(0, 512).unwrap(), + }, + } + } +} impl StoreReader { pub fn metadata( &self, From abfd8cbdc8321ddd45dd9144ed5117a8930b158f Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 05:57:52 +0000 Subject: [PATCH 65/87] docs(racer): describe RACER_GPU_STAGING_BYTES as per-GPU read staging --- deploy/racer/dataplane-config.yaml.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/racer/dataplane-config.yaml.tmpl b/deploy/racer/dataplane-config.yaml.tmpl index 110e25398..ad6c9c85c 100644 --- a/deploy/racer/dataplane-config.yaml.tmpl +++ b/deploy/racer/dataplane-config.yaml.tmpl @@ -17,7 +17,7 @@ data: # mode off when no usable GPU is found; true fails startup instead. # RACER_ENABLE_GPU: "auto" # Cap on ciphertext staging memory per GPU, in bytes: a nonzero multiple of - # 2 MiB, default 256 MiB. Reserved for GPU decryption; not used yet. + # 2 MiB, default 256 MiB. GPU reads stage ciphertext here for decryption. # RACER_GPU_STAGING_BYTES: "268435456" # GPU memory per GPU that peers' NICs write GPU-read pages into over RDMA # (GPUDirect RDMA), in bytes: a multiple of 2 MiB, default 256 MiB. 0 turns From 7110a4dfbbea955939bcbb8c34e7d8bb7d849883 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 06:06:52 +0000 Subject: [PATCH 66/87] refactor(racer): drop dead assignment in vLLM weight stream cleanup --- clients/python/racer-vllm/src/racer_vllm/weights.py | 1 - 1 file changed, 1 deletion(-) diff --git a/clients/python/racer-vllm/src/racer_vllm/weights.py b/clients/python/racer-vllm/src/racer_vllm/weights.py index 767c68e4a..dea0823a8 100644 --- a/clients/python/racer-vllm/src/racer_vllm/weights.py +++ b/clients/python/racer-vllm/src/racer_vllm/weights.py @@ -123,7 +123,6 @@ def fill() -> bool: if active is not None: backend.done(active) backend.wait(active) - active = None # Reads have stopped; fence consumer GPU work before release. Targets # still in flight were fenced by read() and the consumer never saw them. for target in free: From db672baa2b2e582fe87d69c232f8102e0a3002ff Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 06:21:58 +0000 Subject: [PATCH 67/87] fix(racer): stage torch tensor targets on their own GPU --- clients/python/racer/src/racer/tensors.py | 4 ++ clients/python/racer/tests/test_cuda.py | 53 +++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/clients/python/racer/src/racer/tensors.py b/clients/python/racer/src/racer/tensors.py index 79418cb13..be155ed00 100644 --- a/clients/python/racer/src/racer/tensors.py +++ b/clients/python/racer/src/racer/tensors.py @@ -42,6 +42,10 @@ def __init__(self, tensor) -> None: raise ValueError("tensor must be a contiguous 1-D uint8 tensor") self.tensor = tensor self.size = tensor.numel() + # CUDA ordinal of the tensor, so host and stream staging pins memory + # on this GPU rather than device 0. + index = getattr(tensor.device, "index", None) + self.device = index if isinstance(index, int) else None def export_fd(self) -> int: raise NotImplementedError("torch memory cannot be exported") diff --git a/clients/python/racer/tests/test_cuda.py b/clients/python/racer/tests/test_cuda.py index 85b76da1b..2f238bc08 100644 --- a/clients/python/racer/tests/test_cuda.py +++ b/clients/python/racer/tests/test_cuda.py @@ -179,3 +179,56 @@ def test_staging_pinned_for_destination_device(fake, tmp_path): assert ("register", 103) in lib.calls finally: buf.close() + + +def _stub_torch(monkeypatch): + import sys + import types + + torch = types.ModuleType("torch") + torch.uint8 = "uint8" + monkeypatch.setitem(sys.modules, "torch", torch) + + +class _StubTensor: + def __init__(self, index): + self.dtype = "uint8" + self.device = type("Dev", (), {"type": "cuda", "index": index})() + + def dim(self): + return 1 + + def is_contiguous(self): + return True + + def numel(self): + return 16 + + +def test_tensor_target_exposes_device(monkeypatch): + from racer.client import _dest_device + from racer.tensors import TensorTarget + + _stub_torch(monkeypatch) + assert TensorTarget(_StubTensor(3)).device == 3 + assert _dest_device(TensorTarget(_StubTensor(3))) == 3 + assert _dest_device(TensorTarget(_StubTensor(None))) is None + + +def test_tensor_target_staging_on_tensor_device(fake, monkeypatch): + import threading + + from racer.client import Client, _dest_device + from racer.tensors import TensorTarget + + _stub_torch(monkeypatch) + lib = fake(current=0) + c = Client.__new__(Client) + c._pool, c.staging_bytes, c.pin_staging = [], 4096, True + c._pool_lock = threading.Lock() + buf = c._take_staging(_dest_device(TensorTarget(_StubTensor(3)))) + try: + assert ("register", 103) in lib.calls + assert ("register", 100) not in lib.calls + finally: + buf.close() From 8bdbf055da33e03fd3cd546565dfc962226b62c5 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 06:39:37 +0000 Subject: [PATCH 68/87] fix(racer): retain host copy sources until GPU stream sync --- cmd/racer-dataplane/gpu/native/gpu_cuda.h | 6 +- cmd/racer-dataplane/gpu/src/lib.rs | 42 +++++++++---- cmd/racer-dataplane/gpu/tests/simulation.rs | 66 ++++++++++++++++----- cmd/racer-dataplane/src/gpu.rs | 20 ++++++- 4 files changed, 103 insertions(+), 31 deletions(-) diff --git a/cmd/racer-dataplane/gpu/native/gpu_cuda.h b/cmd/racer-dataplane/gpu/native/gpu_cuda.h index 63c2f5a4f..220d6a1bb 100644 --- a/cmd/racer-dataplane/gpu/native/gpu_cuda.h +++ b/cmd/racer-dataplane/gpu/native/gpu_cuda.h @@ -117,8 +117,10 @@ int racer_gpu_import_free(void *allocation); int racer_gpu_staging(void *context, uint64_t size, void **out); int racer_gpu_staging_free(void *staging); -/* Queue a copy of host bytes into an import or staging buffer. The host - * buffer may be reused as soon as this returns. */ +/* Queue a copy of host bytes into an import or staging buffer. The copy is + * asynchronous: for page-locked memory CUDA may still read the host buffer + * after this returns, so the caller must keep it alive and unchanged until + * the stream is synchronized. The Rust wrapper owns the source until then. */ int racer_gpu_copy_h2d(void *stream, void *dst, uint64_t offset, const void *src, uint64_t length); /* Queue a device copy. `src` must be a staging buffer. */ int racer_gpu_copy_d2d(void *stream, void *dst, uint64_t dst_offset, void *src, uint64_t src_offset, diff --git a/cmd/racer-dataplane/gpu/src/lib.rs b/cmd/racer-dataplane/gpu/src/lib.rs index 5c4949fb5..0176fb766 100644 --- a/cmd/racer-dataplane/gpu/src/lib.rs +++ b/cmd/racer-dataplane/gpu/src/lib.rs @@ -495,6 +495,9 @@ enum Retained { Import(#[allow(dead_code)] Rc), Staging(Rc), + + /// Host bytes a queued copy may still read. + Host(#[allow(dead_code)] Box>), } impl Drop for Retained { @@ -612,25 +615,40 @@ impl Stream { self.retained.borrow_mut().extend(holds); } - /// Queue a copy of host bytes. `bytes` may be reused once this returns. - pub fn copy_from_host(&self, dst: Destination<'_>, offset: u64, bytes: &[u8]) -> Result<()> { - if bytes.is_empty() { + /// Queue a copy of host bytes. CUDA may keep reading page-locked host + /// memory after the call returns, so the stream takes ownership of + /// `bytes` and holds it until [`Stream::synchronize`] succeeds (or leaks + /// it if the stream cannot be synchronized). Pass a cheap shared handle, + /// such as an `Arc`, to avoid copying large buffers. + pub fn copy_from_host(&self, dst: Destination<'_>, offset: u64, bytes: S) -> Result<()> + where + S: AsRef<[u8]> + 'static, + { + // Box first so the bytes do not move after the pointer is taken. + let source: Box> = Box::new(bytes); + let slice = (*source).as_ref(); + let (ptr, length) = (slice.as_ptr(), slice.len() as u64); + if length == 0 { return Err(Error::InvalidRange); } - let (raw, hold) = self.destination(dst, offset, bytes.len() as u64)?; + let (raw, hold) = self.destination(dst, offset, length)?; let rc = unsafe { - (self.shared().api.copy_h2d)( - self.raw.as_ptr(), - raw, - offset, - bytes.as_ptr().cast(), - bytes.len() as u64, - ) + (self.shared().api.copy_h2d)(self.raw.as_ptr(), raw, offset, ptr.cast(), length) }; - self.retain([hold]); + self.retain([hold, Retained::Host(source)]); self.shared().check(rc) } + /// Number of host sources held for queued copies. Test helper. + #[doc(hidden)] + pub fn retained_host_sources(&self) -> usize { + self.retained + .borrow() + .iter() + .filter(|hold| matches!(hold, Retained::Host(_))) + .count() + } + /// Queue a device copy from staging memory. pub fn copy_from_staging( &self, diff --git a/cmd/racer-dataplane/gpu/tests/simulation.rs b/cmd/racer-dataplane/gpu/tests/simulation.rs index 7a4bc3c50..b406b24cb 100644 --- a/cmd/racer-dataplane/gpu/tests/simulation.rs +++ b/cmd/racer-dataplane/gpu/tests/simulation.rs @@ -139,25 +139,25 @@ fn import_rejects_out_of_window_and_wrong_device() { let stream = ctx.stream().unwrap(); let foreign = other.stream().unwrap(); let dst = Destination::Import(&import); - stream.copy_from_host(dst, 100, &[7; 100]).unwrap(); + stream.copy_from_host(dst, 100, [7; 100]).unwrap(); assert_eq!( - stream.copy_from_host(dst, 99, &[1; 2]), + stream.copy_from_host(dst, 99, [1; 2]), Err(Error::InvalidRange) ); assert_eq!( - stream.copy_from_host(dst, 150, &[1; 51]), + stream.copy_from_host(dst, 150, [1; 51]), Err(Error::InvalidRange) ); assert_eq!( - stream.copy_from_host(dst, u64::MAX, &[1]), + stream.copy_from_host(dst, u64::MAX, [1]), Err(Error::InvalidRange) ); assert_eq!( - stream.copy_from_host(dst, 100, &[]), + stream.copy_from_host(dst, 100, []), Err(Error::InvalidRange) ); assert_eq!( - foreign.copy_from_host(dst, 100, &[1]), + foreign.copy_from_host(dst, 100, [1]), Err(Error::WrongDevice) ); stream.synchronize().unwrap(); @@ -187,7 +187,7 @@ fn decrypt_into_import_verifies_before_writing() { assert_eq!(stream.verdict(), Err(Error::Busy)); stream - .copy_from_host(Destination::Staging(&staging), 16, &page.sealed) + .copy_from_host(Destination::Staging(&staging), 16, page.sealed.clone()) .unwrap(); stream .decrypt(&page.request(Some(crc)), &staging, 16, dst, 64) @@ -217,7 +217,7 @@ fn decrypt_into_import_verifies_before_writing() { let mut tampered = page.sealed.clone(); tampered[4999] ^= 1; stream - .copy_from_host(Destination::Staging(&staging), 16, &tampered) + .copy_from_host(Destination::Staging(&staging), 16, tampered) .unwrap(); stream .decrypt(&page.request(None), &staging, 16, dst2, 0) @@ -286,7 +286,7 @@ fn import_is_released_only_after_stream_sync() { let import = ctx.import(client, GRANULARITY, 0..10).unwrap(); let stream = ctx.stream().unwrap(); stream - .copy_from_host(Destination::Import(&import), 0, b"0123456789") + .copy_from_host(Destination::Import(&import), 0, *b"0123456789") .unwrap(); drop(import); assert_eq!(sim.live().imports, 1); @@ -304,7 +304,7 @@ fn import_is_released_only_after_stream_sync() { let client = sim.allocate(0, GRANULARITY); let import = ctx.import(client, GRANULARITY, 0..10).unwrap(); stream - .copy_from_host(Destination::Import(&import), 0, b"x") + .copy_from_host(Destination::Import(&import), 0, *b"x") .unwrap(); drop(import); drop(stream); @@ -333,7 +333,7 @@ fn failed_frees_leak_and_count() { let stream = ctx.stream().unwrap(); let staging = ctx.staging(16).unwrap(); stream - .copy_from_host(Destination::Staging(&staging), 0, &[1; 16]) + .copy_from_host(Destination::Staging(&staging), 0, [1; 16]) .unwrap(); assert!(staging.in_flight()); sim.fault(Operation::StreamFree, -5); @@ -347,7 +347,7 @@ fn failed_frees_leak_and_count() { let stream = ctx.stream().unwrap(); let staging = ctx.staging(16).unwrap(); stream - .copy_from_host(Destination::Staging(&staging), 0, &[1; 16]) + .copy_from_host(Destination::Staging(&staging), 0, [1; 16]) .unwrap(); sim.fault(Operation::StreamSync, -5); assert_eq!(stream.synchronize(), Err(Error::Driver(999))); @@ -384,7 +384,7 @@ fn cufile_reads_into_staging() { // A queued copy holds the buffer. stream - .copy_from_host(Destination::Staging(&staging), 0, &[1]) + .copy_from_host(Destination::Staging(&staging), 0, [1]) .unwrap(); assert_eq!(file.read(&staging, 0, 0, 1), Err(Error::Busy)); stream.synchronize().unwrap(); @@ -431,7 +431,7 @@ fn dma_buf_export_from_zero_is_live() { stream.synchronize().unwrap(); assert_eq!(&out, b"from the nic"); stream - .copy_from_host(Destination::Staging(&staging), 4096, b"gpu") + .copy_from_host(Destination::Staging(&staging), 4096, *b"gpu") .unwrap(); stream.synchronize().unwrap(); let mut back = [0; 3]; @@ -455,7 +455,7 @@ fn device_copy_from_staging() { let staging = ctx.staging(32).unwrap(); let stream = ctx.stream().unwrap(); stream - .copy_from_host(Destination::Staging(&staging), 0, b"abcdefghij") + .copy_from_host(Destination::Staging(&staging), 0, *b"abcdefghij") .unwrap(); stream .copy_from_staging(Destination::Import(&import), 10, &staging, 0, 10) @@ -471,3 +471,39 @@ fn device_copy_from_staging() { stream.synchronize().unwrap(); assert_eq!(read_allocation(&client, 10, 10), b"abcdefghij"); } + +#[test] +fn host_source_is_held_until_stream_sync() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let client = sim.allocate(0, GRANULARITY); + let import = ctx + .import(client.try_clone().unwrap(), GRANULARITY, 0..10) + .unwrap(); + let stream = ctx.stream().unwrap(); + let source = std::sync::Arc::<[u8]>::from(&b"0123456789"[..]); + let weak = std::sync::Arc::downgrade(&source); + stream + .copy_from_host(Destination::Import(&import), 0, source.clone()) + .unwrap(); + // The caller drops its handle early; the stream still owns the bytes. + drop(source); + assert_eq!(weak.strong_count(), 1); + assert_eq!(stream.retained_host_sources(), 1); + stream.synchronize().unwrap(); + assert_eq!(weak.strong_count(), 0); + assert_eq!(stream.retained_host_sources(), 0); + assert_eq!(read_allocation(&client, 0, 10), b"0123456789"); + + // A rejected copy does not keep the source. + let source = std::sync::Arc::<[u8]>::from(vec![1u8; 11]); + let weak = std::sync::Arc::downgrade(&source); + assert_eq!( + stream.copy_from_host(Destination::Import(&import), 0, source), + Err(Error::InvalidRange) + ); + assert_eq!(weak.strong_count(), 0); + assert_eq!(stream.retained_host_sources(), 0); +} diff --git a/cmd/racer-dataplane/src/gpu.rs b/cmd/racer-dataplane/src/gpu.rs index 31eeb1c30..70b700ae0 100644 --- a/cmd/racer-dataplane/src/gpu.rs +++ b/cmd/racer-dataplane/src/gpu.rs @@ -99,17 +99,33 @@ const PLAINTEXT_SCRATCH: u64 = SLOT_BYTES; /// Registered cuFile handles kept per process; more are reopened on demand. const MAX_FILES: usize = 64; +/// A shared ciphertext page the GPU stream holds until its copy completes. +struct HostCiphertext(crate::memory::CiphertextPage); + +impl AsRef<[u8]> for HostCiphertext { + fn as_ref(&self) -> &[u8] { + self.0.bytes() + } +} + /// Builds the GPU backend on the helper thread. pub type Loader = Box racer_gpu::Result + Send>; /// Plaintext bytes handed to the helper. The page is immutable and its /// storage is reference counted, so the helper reads it in place; the handle /// goes back with the answer so the page is released on the I/O thread. +#[derive(Clone)] struct Plaintext { page: VerifiedPage, range: Range, } +impl AsRef<[u8]> for Plaintext { + fn as_ref(&self) -> &[u8] { + self.bytes() + } +} + impl Plaintext { fn bytes(&self) -> &[u8] { &self.page.bytes()[self.range.clone()] @@ -1062,7 +1078,7 @@ impl Helper { let destination = racer_gpu::Destination::Import(&request.import); let queued = request .stream - .copy_from_host(destination, offset, plaintext.bytes()); + .copy_from_host(destination, offset, plaintext.clone()); // Synchronize even after a failed queue call: part of it may run. let synchronized = request.stream.synchronize(); queued.and(synchronized).map_err(gpu_error) @@ -1170,7 +1186,7 @@ impl Helper { let copied = stream.copy_from_host( racer_gpu::Destination::Staging(staging), CIPHERTEXT_SCRATCH, - bytes, + HostCiphertext(copy.copy.ciphertext.clone()), ); if let Err(error) = copied { let _ = stream.synchronize(); From 8f3e79639ba008fba3cb52c84a306c2b735b66ca Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 06:36:57 +0000 Subject: [PATCH 69/87] fix(racer): reject non-integer safetensors shapes and offsets --- .../racer-vllm/src/racer_vllm/safetensors.py | 15 ++++++++++++--- .../python/racer-vllm/tests/test_safetensors.py | 11 +++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/clients/python/racer-vllm/src/racer_vllm/safetensors.py b/clients/python/racer-vllm/src/racer_vllm/safetensors.py index 88d20cffa..1d92fb17b 100644 --- a/clients/python/racer-vllm/src/racer_vllm/safetensors.py +++ b/clients/python/racer-vllm/src/racer_vllm/safetensors.py @@ -74,6 +74,11 @@ def header_length(prefix: bytes) -> int: return n +def _is_uint_list(v) -> bool: + # JSON integers only: reject floats, numeric strings, and bools (bool is an int subclass). + return isinstance(v, list) and all(type(x) is int and x >= 0 for x in v) + + def parse_header(raw: bytes, file_size: int | None = None) -> Header: """Parse header JSON (without the 8-byte prefix) into sorted TensorInfo.""" try: @@ -88,10 +93,14 @@ def parse_header(raw: bytes, file_size: int | None = None) -> Header: for name, spec in doc.items(): try: dtype = spec["dtype"] - shape = tuple(int(d) for d in spec["shape"]) - begin, end = (int(x) for x in spec["data_offsets"]) - except (KeyError, TypeError, ValueError) as exc: + raw_shape = spec["shape"] + raw_offsets = spec["data_offsets"] + except (KeyError, TypeError) as exc: raise SafetensorsError(f"bad entry for {name!r}") from exc + if not _is_uint_list(raw_shape) or not _is_uint_list(raw_offsets) or len(raw_offsets) != 2: + raise SafetensorsError(f"bad shape or offsets for {name!r}") + shape = tuple(raw_shape) + begin, end = raw_offsets if dtype not in DTYPE_SIZES: raise SafetensorsError(f"unsupported dtype {dtype} for {name!r}") if any(d < 0 for d in shape) or begin < 0 or end < begin: diff --git a/clients/python/racer-vllm/tests/test_safetensors.py b/clients/python/racer-vllm/tests/test_safetensors.py index 750b8b685..9648e3762 100644 --- a/clients/python/racer-vllm/tests/test_safetensors.py +++ b/clients/python/racer-vllm/tests/test_safetensors.py @@ -66,6 +66,17 @@ def test_plan_chunks_split_and_dedicated(): b'{"a":{"dtype":"U8","shape":[2],"data_offsets":[0,2]},"b":{"dtype":"U8","shape":[2],"data_offsets":[1,3]}}', b'{"a":{"dtype":"U8","shape":[-1],"data_offsets":[0,0]}}', b'{"a":{"dtype":"U8"}}', + b'{"a":{"dtype":"U8","shape":[1.9],"data_offsets":[0.9,1.9]}}', + b'{"a":{"dtype":"U8","shape":[1],"data_offsets":[0.0,1.0]}}', + b'{"a":{"dtype":"U8","shape":[-0.5],"data_offsets":[0,0]}}', + b'{"a":{"dtype":"U8","shape":["1"],"data_offsets":[0,1]}}', + b'{"a":{"dtype":"U8","shape":[1],"data_offsets":["0","1"]}}', + b'{"a":{"dtype":"U8","shape":[true],"data_offsets":[0,1]}}', + b'{"a":{"dtype":"U8","shape":[1],"data_offsets":[false,true]}}', + b'{"a":{"dtype":"U8","shape":"1","data_offsets":[0,1]}}', + b'{"a":{"dtype":"U8","shape":[1],"data_offsets":[0,1,2]}}', + b'{"a":{"dtype":"U8","shape":[1],"data_offsets":{"0":0}}}', + b'{"a":[]}', ], ) def test_bad_headers(header): From 032f4e3bbcce99a9d3272479e004ae44cf011a3e Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 06:53:07 +0000 Subject: [PATCH 70/87] fix(racer): release host registration exports when pinning fails --- clients/python/racer/src/racer/cuda.py | 23 +++++++++++++--------- clients/python/racer/src/racer/host.py | 10 ++++++---- clients/python/racer/tests/test_cuda.py | 26 +++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/clients/python/racer/src/racer/cuda.py b/clients/python/racer/src/racer/cuda.py index e56bad471..122aff882 100644 --- a/clients/python/racer/src/racer/cuda.py +++ b/clients/python/racer/src/racer/cuda.py @@ -412,13 +412,18 @@ def host_register(buf, device: int = 0) -> HostRegistration: """ lib = _load() mv = memoryview(buf).cast("B") - if mv.readonly: - raise RacerError("buffer must be writable") - arr = (ctypes.c_char * len(mv)).from_buffer(mv) - addr = ctypes.addressof(arr) - with bound_context(device): - _check("cuMemHostRegister_v2", lib.cuMemHostRegister_v2(addr, len(mv), CU_MEMHOSTREGISTER_PORTABLE)) - # Drop the export so the owner can still close its mapping after unregistering. - del arr - mv.release() + arr = None + try: + if mv.readonly: + raise RacerError("buffer must be writable") + n = len(mv) + arr = (ctypes.c_char * n).from_buffer(mv) + addr = ctypes.addressof(arr) + with bound_context(device): + _check("cuMemHostRegister_v2", lib.cuMemHostRegister_v2(addr, n, CU_MEMHOSTREGISTER_PORTABLE)) + finally: + # Drop the ctypes export and the view even on failure, so a traceback + # that references this frame cannot keep the owner's mapping exported. + arr = None + mv.release() return HostRegistration(addr, None) diff --git a/clients/python/racer/src/racer/host.py b/clients/python/racer/src/racer/host.py index de24e51d0..90d1fe49b 100644 --- a/clients/python/racer/src/racer/host.py +++ b/clients/python/racer/src/racer/host.py @@ -59,10 +59,12 @@ def close(self) -> None: self._pinned.close() self._pinned = None if self._fd >= 0: - self.view.release() - self._map.close() - os.close(self._fd) - self._fd = -1 + fd, self._fd = self._fd, -1 + try: + self.view.release() + self._map.close() + finally: + os.close(fd) def __enter__(self) -> "HostBuffer": return self diff --git a/clients/python/racer/tests/test_cuda.py b/clients/python/racer/tests/test_cuda.py index 2f238bc08..406803abe 100644 --- a/clients/python/racer/tests/test_cuda.py +++ b/clients/python/racer/tests/test_cuda.py @@ -165,6 +165,32 @@ def test_host_register_restores_context_on_error(fake): assert lib.current == 100 +def test_failed_pin_releases_exports_and_closes_memfd(fake): + import os + + from racer import HostBuffer + + fake(fail="cuMemHostRegister_v2") + with pytest.raises(cuda.CudaError) as ei: + with HostBuffer(4096) as b: + fd = b._fd + b.pin(0) + assert "cuMemHostRegister_v2" in str(ei.value) + assert b._map.closed + with pytest.raises(OSError): + os.fstat(fd) + + +def test_host_register_releases_export_on_failure(fake): + import mmap + + fake(fail="cuMemHostRegister_v2") + m = mmap.mmap(-1, 4096) + with pytest.raises(cuda.CudaError): + cuda.host_register(m, 0) + m.close() + + def test_staging_pinned_for_destination_device(fake, tmp_path): from racer.client import Client From 41f8f6ee1737b37daa82238134de0124aa386090 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 06:57:14 +0000 Subject: [PATCH 71/87] refactor(racer): drop the host registration ctypes export explicitly --- clients/python/racer/src/racer/cuda.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clients/python/racer/src/racer/cuda.py b/clients/python/racer/src/racer/cuda.py index 122aff882..0ab7e3e39 100644 --- a/clients/python/racer/src/racer/cuda.py +++ b/clients/python/racer/src/racer/cuda.py @@ -424,6 +424,7 @@ def host_register(buf, device: int = 0) -> HostRegistration: finally: # Drop the ctypes export and the view even on failure, so a traceback # that references this frame cannot keep the owner's mapping exported. - arr = None + # The ctypes array holds an export of mv, so it must go before release. + del arr mv.release() return HostRegistration(addr, None) From 6b20bc81e0c21d171b72bbc6fd245a194237674e Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 07:15:29 +0000 Subject: [PATCH 72/87] fix(racer): attach received descriptors to the request head they arrived with A descriptor-aware head read could return a plain HEAD and the start of a following descriptor-bearing request from one recvmsg. Every descriptor was attached to the first decoded head, so the HEAD was rejected for carrying an unexpected descriptor and neither request was served. Linux unix_stream_read_generic merges skbs into one stream read but stops after an skb carrying SCM_RIGHTS. A client sends a head's descriptors with the start of that head, so descriptors from one receive belong to the last head that starts in its bytes. The receive loop now keeps the newest chunk's descriptors apart: earlier chunks belong to the current head, and the last chunk does too unless bytes remain after the head ends. Otherwise they are carried on the lease with the read-ahead and the same rule is applied when the next head is decoded from it. Carried descriptors still count toward MAX_HEAD_DESCRIPTORS, close with the lease or a rejected head, and a non-descriptor head read that finds them closes them and rejects. The http layer still rejects pipelining at exchange completion, so a pipelined host-read after a HEAD now gets the HEAD served and the connection closed with the carried descriptor released. Also make the client disconnect cancellation test pump until the hangup is observed instead of a fixed pump count; it failed under parallel load. --- .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../added/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../.client-test-2180946-68/blocked | 1 + .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/origin/socket | 1 + .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../other/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../added/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/origin/socket | 1 + .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../.client-test-2192245-82/ready | 1 + .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../other/client/.racer-client.lock | 0 .../added/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../.client-test-2193402-74/blocked | 1 + .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../.client-test-2193712-67/blocked | 1 + .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../.client-test-2193981-81/ready | 1 + .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../other/client/.racer-client.lock | 0 .../added/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../added/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../.client-test-2194756-83/ready | 1 + .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../added/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../other/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/origin/socket | 1 + .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../.client-test-2195404-79/ready | 1 + .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 .../example/client/.racer-client.lock | 0 cmd/racer-dataplane/http/src/connection.rs | 64 ++++++++++++-- cmd/racer-dataplane/http/tests/exchanges.rs | 86 +++++++++++++++++++ cmd/racer-dataplane/src/client/tests.rs | 9 +- .../src/client/tests/direct.rs | 31 +++++++ 555 files changed, 194 insertions(+), 6 deletions(-) create mode 100644 cmd/racer-dataplane/.client-test-2179449-11/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-12/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-13/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-15/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-17/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-18/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-20/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-21/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-22/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-23/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-24/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-25/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-33/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-34/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-37/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-38/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-39/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-4/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-40/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-42/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-46/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-5/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-57/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-58/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-59/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-60/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-61/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-65/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-69/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-8/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179449-9/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-10/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-11/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-12/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-13/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-14/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-16/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-18/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-2/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-20/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-21/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-22/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-23/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-24/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-25/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-26/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-29/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-30/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-33/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-35/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-36/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-37/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-38/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-39/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-4/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-41/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-42/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-43/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-5/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-52/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-6/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-60/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-62/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-7/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-8/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2179709-9/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-10/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-11/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-14/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-17/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-18/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-20/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-21/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-22/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-25/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-26/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-28/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-29/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-33/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-36/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-38/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-39/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-4/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-42/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-47/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-57/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-58/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-61/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-62/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-63/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-67/added/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-67/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-68/blocked create mode 100644 cmd/racer-dataplane/.client-test-2180946-68/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-69/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-69/example/origin/socket create mode 100644 cmd/racer-dataplane/.client-test-2180946-70/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-71/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-75/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-8/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2180946-9/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-10/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-11/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-16/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-17/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-18/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-2/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-20/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-21/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-22/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-23/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-25/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-26/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-28/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-29/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-31/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-33/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-34/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-39/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-4/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-41/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-42/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-43/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-45/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-5/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-57/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-58/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-59/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-6/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-60/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-7/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-8/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181225-9/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-10/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-11/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-16/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-17/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-18/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-2/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-21/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-22/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-23/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-24/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-25/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-27/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-28/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-30/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-33/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-34/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-35/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-36/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-37/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-38/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-39/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-4/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-40/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-41/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-44/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-55/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-56/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-59/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-6/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-63/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-63/other/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-65/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-67/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-8/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2181449-9/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-10/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-13/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-14/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-16/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-17/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-18/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-2/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-21/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-22/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-23/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-24/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-25/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-26/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-28/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-29/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-33/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-34/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-35/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-36/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-38/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-39/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-4/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-40/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-41/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-43/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-47/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-5/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-8/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2182990-9/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-10/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-11/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-12/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-14/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-16/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-17/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-18/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-2/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-20/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-23/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-24/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-25/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-26/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-28/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-33/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-34/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-35/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-38/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-39/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-4/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-41/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-42/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-5/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-6/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-60/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-62/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-7/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-8/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2183207-9/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-11/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-12/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-14/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-15/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-17/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-18/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-20/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-22/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-23/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-28/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-29/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-30/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-31/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-33/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-36/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-37/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-38/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-4/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-40/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-41/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-42/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-43/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-45/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-5/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-54/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-55/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-6/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-8/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184396-9/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-11/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-14/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-16/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-17/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-18/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-2/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-21/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-22/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-23/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-27/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-32/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-33/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-34/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-35/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-36/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-4/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-40/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-41/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-43/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-45/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-5/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-57/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-59/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-60/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-61/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-64/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2184635-68/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-102/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-105/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-14/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-15/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-16/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-20/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-22/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-24/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-36/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-37/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-38/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-41/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-46/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-47/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-48/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-5/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-51/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-54/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-58/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-61/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-63/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-65/added/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-65/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-67/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-67/example/origin/socket create mode 100644 cmd/racer-dataplane/.client-test-2192245-76/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-82/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-82/ready create mode 100644 cmd/racer-dataplane/.client-test-2192245-91/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-96/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2192245-98/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-16/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-17/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-18/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-21/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-25/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-39/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-4/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-42/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-43/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-57/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-60/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-62/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-65/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-66/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-67/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-69/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-71/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-72/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-72/other/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-73/added/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-73/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-74/blocked create mode 100644 cmd/racer-dataplane/.client-test-2193402-74/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-78/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-79/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-82/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193402-84/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-14/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-16/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-17/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-18/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-21/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-22/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-23/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-27/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-28/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-30/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-38/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-41/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-42/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-43/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-44/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-45/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-49/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-54/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-57/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-6/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-60/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-63/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-65/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-67/blocked create mode 100644 cmd/racer-dataplane/.client-test-2193712-67/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-68/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-69/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193712-70/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193981-14/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193981-17/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193981-18/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193981-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193981-21/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193981-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193981-37/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193981-44/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193981-54/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193981-55/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193981-73/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193981-80/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193981-81/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193981-81/ready create mode 100644 cmd/racer-dataplane/.client-test-2193981-82/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193981-83/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2193981-86/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-10/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-11/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-12/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-14/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-15/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-16/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-17/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-20/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-22/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-23/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-28/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-31/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-34/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-35/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-36/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-38/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-4/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-40/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-41/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-45/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-5/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-7/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194305-8/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-10/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-12/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-13/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-14/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-16/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-17/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-18/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-20/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-21/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-23/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-26/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-27/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-28/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-29/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-31/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-33/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-34/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-35/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-37/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-4/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-41/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-42/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-43/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-45/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-57/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-58/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-59/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-61/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-64/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-65/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-66/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-66/other/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-67/added/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-67/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-69/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-8/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194507-9/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-100/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-102/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-14/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-17/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-18/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-20/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-31/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-33/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-34/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-35/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-38/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-41/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-42/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-59/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-60/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-62/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-66/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-68/added/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-83/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-83/ready create mode 100644 cmd/racer-dataplane/.client-test-2194756-85/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-94/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2194756-97/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-14/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-17/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-18/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-21/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-25/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-26/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-27/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-28/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-29/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-30/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-33/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-35/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-36/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-38/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-4/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-42/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-45/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-5/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-59/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-61/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-63/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-64/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-66/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-67/added/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-67/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-68/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-68/other/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-70/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-71/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-71/example/origin/socket create mode 100644 cmd/racer-dataplane/.client-test-2195107-80/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195107-82/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-100/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-102/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-104/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-13/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-16/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-17/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-19/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-21/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-3/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-33/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-46/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-54/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-58/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-79/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-79/ready create mode 100644 cmd/racer-dataplane/.client-test-2195404-80/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-83/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-87/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-93/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-95/example/client/.racer-client.lock create mode 100644 cmd/racer-dataplane/.client-test-2195404-97/example/client/.racer-client.lock diff --git a/cmd/racer-dataplane/.client-test-2179449-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-11/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-12/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-12/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-13/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-13/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-15/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-15/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-17/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-18/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-20/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-21/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-22/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-23/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-24/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-24/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-25/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-33/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-34/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-37/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-37/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-38/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-39/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-39/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-4/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-40/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-40/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-42/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-46/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-46/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-5/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-57/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-57/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-58/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-58/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-59/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-59/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-60/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-60/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-61/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-61/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-65/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-65/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-69/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-69/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-8/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179449-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179449-9/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-10/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-10/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-11/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-12/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-12/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-13/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-13/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-14/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-16/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-18/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-2/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-2/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-20/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-21/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-22/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-23/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-24/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-24/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-25/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-26/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-26/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-29/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-29/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-30/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-30/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-33/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-35/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-36/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-37/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-37/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-38/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-39/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-39/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-4/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-41/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-42/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-43/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-43/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-5/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-52/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-52/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-6/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-6/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-60/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-60/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-62/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-62/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-7/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-7/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-8/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2179709-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2179709-9/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-10/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-10/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-11/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-14/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-17/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-18/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-20/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-21/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-22/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-25/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-26/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-26/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-28/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-29/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-29/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-33/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-36/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-38/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-39/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-39/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-4/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-42/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-47/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-47/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-57/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-57/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-58/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-58/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-61/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-61/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-62/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-62/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-63/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-63/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-67/added/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-67/added/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-67/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-67/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-68/blocked b/cmd/racer-dataplane/.client-test-2180946-68/blocked new file mode 100644 index 000000000..2a5e26918 --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2180946-68/blocked @@ -0,0 +1 @@ +foreign \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2180946-68/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-68/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-69/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-69/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-69/example/origin/socket b/cmd/racer-dataplane/.client-test-2180946-69/example/origin/socket new file mode 100644 index 000000000..6fa78783b --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2180946-69/example/origin/socket @@ -0,0 +1 @@ +origin \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2180946-70/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-70/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-71/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-71/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-75/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-75/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-8/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2180946-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2180946-9/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-10/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-10/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-11/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-16/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-17/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-18/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-2/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-2/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-20/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-21/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-22/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-23/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-25/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-26/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-26/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-28/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-29/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-29/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-31/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-31/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-33/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-34/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-39/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-39/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-4/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-41/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-42/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-43/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-43/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-45/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-45/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-5/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-57/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-57/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-58/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-58/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-59/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-59/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-6/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-6/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-60/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-60/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-7/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-7/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-8/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181225-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181225-9/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-10/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-10/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-11/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-16/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-17/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-18/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-2/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-2/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-21/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-22/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-23/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-24/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-24/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-25/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-27/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-27/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-28/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-30/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-30/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-33/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-34/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-35/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-36/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-37/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-37/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-38/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-39/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-39/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-4/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-40/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-40/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-41/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-44/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-44/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-55/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-55/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-56/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-56/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-59/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-59/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-6/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-6/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-63/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-63/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-63/other/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-63/other/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-65/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-65/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-67/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-67/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-8/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2181449-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2181449-9/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-10/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-10/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-13/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-13/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-14/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-16/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-17/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-18/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-2/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-2/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-21/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-22/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-23/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-24/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-24/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-25/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-26/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-26/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-28/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-29/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-29/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-33/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-34/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-35/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-36/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-38/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-39/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-39/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-4/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-40/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-40/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-41/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-43/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-43/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-47/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-47/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-5/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-8/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2182990-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2182990-9/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-10/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-10/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-11/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-12/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-12/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-14/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-16/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-17/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-18/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-2/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-2/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-20/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-23/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-24/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-24/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-25/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-26/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-26/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-28/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-33/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-34/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-35/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-38/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-39/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-39/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-4/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-41/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-42/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-5/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-6/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-6/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-60/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-60/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-62/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-62/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-7/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-7/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-8/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2183207-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2183207-9/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-11/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-12/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-12/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-14/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-15/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-15/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-17/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-18/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-20/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-22/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-23/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-28/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-29/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-29/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-30/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-30/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-31/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-31/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-33/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-36/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-37/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-37/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-38/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-4/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-40/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-40/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-41/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-42/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-43/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-43/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-45/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-45/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-5/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-54/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-54/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-55/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-55/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-6/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-6/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-8/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184396-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184396-9/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-11/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-14/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-16/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-17/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-18/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-2/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-2/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-21/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-22/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-23/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-27/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-27/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-32/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-32/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-33/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-34/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-35/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-36/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-4/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-40/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-40/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-41/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-43/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-43/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-45/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-45/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-5/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-57/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-57/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-59/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-59/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-60/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-60/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-61/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-61/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-64/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-64/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2184635-68/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2184635-68/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-102/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-102/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-105/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-105/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-14/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-15/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-15/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-16/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-20/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-22/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-24/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-24/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-36/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-37/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-37/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-38/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-41/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-46/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-46/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-47/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-47/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-48/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-48/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-5/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-51/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-51/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-54/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-54/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-58/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-58/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-61/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-61/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-63/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-63/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-65/added/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-65/added/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-65/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-65/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-67/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-67/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-67/example/origin/socket b/cmd/racer-dataplane/.client-test-2192245-67/example/origin/socket new file mode 100644 index 000000000..6fa78783b --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2192245-67/example/origin/socket @@ -0,0 +1 @@ +origin \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2192245-76/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-76/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-82/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-82/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-82/ready b/cmd/racer-dataplane/.client-test-2192245-82/ready new file mode 100644 index 000000000..ad50b5340 --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2192245-82/ready @@ -0,0 +1 @@ +ready \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2192245-91/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-91/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-96/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-96/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2192245-98/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2192245-98/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-16/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-17/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-18/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-21/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-25/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-39/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-39/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-4/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-42/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-43/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-43/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-57/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-57/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-60/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-60/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-62/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-62/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-65/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-65/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-66/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-66/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-67/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-67/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-69/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-69/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-71/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-71/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-72/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-72/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-72/other/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-72/other/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-73/added/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-73/added/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-73/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-73/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-74/blocked b/cmd/racer-dataplane/.client-test-2193402-74/blocked new file mode 100644 index 000000000..2a5e26918 --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2193402-74/blocked @@ -0,0 +1 @@ +foreign \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2193402-74/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-74/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-78/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-78/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-79/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-79/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-82/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-82/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193402-84/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193402-84/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-14/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-16/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-17/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-18/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-21/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-22/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-23/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-27/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-27/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-28/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-30/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-30/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-38/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-41/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-42/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-43/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-43/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-44/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-44/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-45/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-45/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-49/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-49/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-54/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-54/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-57/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-57/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-6/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-6/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-60/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-60/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-63/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-63/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-65/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-65/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-67/blocked b/cmd/racer-dataplane/.client-test-2193712-67/blocked new file mode 100644 index 000000000..2a5e26918 --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2193712-67/blocked @@ -0,0 +1 @@ +foreign \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2193712-67/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-67/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-68/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-68/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-69/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-69/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193712-70/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193712-70/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193981-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-14/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193981-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-17/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193981-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-18/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193981-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193981-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-21/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193981-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193981-37/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-37/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193981-44/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-44/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193981-54/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-54/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193981-55/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-55/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193981-73/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-73/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193981-80/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-80/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193981-81/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-81/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193981-81/ready b/cmd/racer-dataplane/.client-test-2193981-81/ready new file mode 100644 index 000000000..ad50b5340 --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2193981-81/ready @@ -0,0 +1 @@ +ready \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2193981-82/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-82/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193981-83/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-83/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2193981-86/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2193981-86/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-10/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-10/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-11/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-11/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-12/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-12/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-14/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-15/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-15/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-16/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-17/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-20/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-22/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-22/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-23/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-28/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-31/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-31/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-34/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-35/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-36/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-38/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-4/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-40/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-40/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-41/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-45/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-45/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-5/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-7/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-7/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194305-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194305-8/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-10/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-10/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-12/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-12/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-13/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-13/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-14/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-16/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-17/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-18/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-20/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-21/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-23/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-23/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-26/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-26/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-27/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-27/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-28/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-29/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-29/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-31/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-31/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-33/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-34/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-35/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-37/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-37/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-4/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-41/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-42/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-43/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-43/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-45/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-45/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-57/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-57/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-58/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-58/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-59/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-59/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-61/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-61/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-64/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-64/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-65/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-65/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-66/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-66/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-66/other/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-66/other/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-67/added/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-67/added/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-67/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-67/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-69/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-69/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-8/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-8/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194507-9/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194507-9/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-100/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-100/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-102/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-102/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-14/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-17/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-18/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-20/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-20/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-31/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-31/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-33/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-34/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-34/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-35/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-38/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-41/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-41/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-42/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-59/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-59/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-60/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-60/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-62/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-62/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-66/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-66/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-68/added/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-68/added/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-83/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-83/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-83/ready b/cmd/racer-dataplane/.client-test-2194756-83/ready new file mode 100644 index 000000000..ad50b5340 --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2194756-83/ready @@ -0,0 +1 @@ +ready \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2194756-85/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-85/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-94/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-94/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2194756-97/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2194756-97/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-14/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-14/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-17/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-18/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-18/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-21/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-25/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-25/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-26/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-26/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-27/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-27/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-28/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-28/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-29/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-29/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-30/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-30/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-33/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-35/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-35/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-36/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-36/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-38/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-38/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-4/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-4/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-42/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-42/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-45/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-45/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-5/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-5/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-59/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-59/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-61/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-61/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-63/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-63/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-64/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-64/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-66/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-66/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-67/added/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-67/added/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-67/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-67/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-68/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-68/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-68/other/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-68/other/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-70/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-70/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-71/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-71/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-71/example/origin/socket b/cmd/racer-dataplane/.client-test-2195107-71/example/origin/socket new file mode 100644 index 000000000..6fa78783b --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2195107-71/example/origin/socket @@ -0,0 +1 @@ +origin \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2195107-80/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-80/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195107-82/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195107-82/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-100/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-100/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-102/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-102/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-104/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-104/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-13/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-13/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-16/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-16/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-17/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-17/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-19/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-19/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-21/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-21/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-3/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-3/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-33/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-33/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-46/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-46/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-54/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-54/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-58/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-58/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-79/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-79/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-79/ready b/cmd/racer-dataplane/.client-test-2195404-79/ready new file mode 100644 index 000000000..ad50b5340 --- /dev/null +++ b/cmd/racer-dataplane/.client-test-2195404-79/ready @@ -0,0 +1 @@ +ready \ No newline at end of file diff --git a/cmd/racer-dataplane/.client-test-2195404-80/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-80/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-83/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-83/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-87/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-87/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-93/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-93/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-95/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-95/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/.client-test-2195404-97/example/client/.racer-client.lock b/cmd/racer-dataplane/.client-test-2195404-97/example/client/.racer-client.lock new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/racer-dataplane/http/src/connection.rs b/cmd/racer-dataplane/http/src/connection.rs index 16bba406d..fe28f8097 100644 --- a/cmd/racer-dataplane/http/src/connection.rs +++ b/cmd/racer-dataplane/http/src/connection.rs @@ -300,6 +300,10 @@ pub struct ConnectionLease { read_ahead: Option<(OwnedBuffer, Range)>, + /// Descriptors from the receive that produced `read_ahead`. They belong to + /// a later request head that starts inside those bytes. + ahead_descriptors: ReceivedDescriptors, + rx_remaining: Option, tx_remaining: Option, @@ -335,6 +339,27 @@ pub struct ReceivedDescriptors { pub truncated: bool, } +impl ReceivedDescriptors { + /// Whether any descriptor or truncation was recorded. + fn has_any(&self) -> bool { + self.truncated || !self.descriptors.is_empty() + } + + /// Move `other` into `self`, closing descriptors beyond the head cap. + fn absorb(&mut self, other: ReceivedDescriptors) { + self.truncated |= other.truncated; + for descriptor in other.descriptors { + if self.descriptors.len() < MAX_HEAD_DESCRIPTORS { + self.descriptors.push(descriptor); + } else { + // Over the aggregate cap: close now and reject the head. + drop(descriptor); + self.truncated = true; + } + } + } +} + /// A decoded request head and the descriptors that arrived with it. pub struct RequestHead { /// Admitted request head. @@ -648,6 +673,20 @@ impl HttpIo { let mut buffer = self.buffer(codec.header_limit().min(4096.max(ahead_length)))?; let mut used = 0; let mut scan = HeadScan::default(); + // Descriptors of the most recent receive (or of the carried + // read-ahead). Linux unix_stream_read_generic (net/unix/af_unix.c) + // may merge several skbs into one recvmsg but stops right after an + // skb carrying SCM_RIGHTS (`if (scm.fp) break;`). A client sends a + // head's descriptors with the start of that head, so descriptors + // returned by one receive belong to the last head starting in its + // bytes. Earlier chunks always belong to the current head; the + // last chunk does too unless bytes remain after this head ends. + let mut last = std::mem::take(&mut connection.ahead_descriptors); + if ancillary.is_none() && last.has_any() { + // The caller does not accept descriptors: close and reject. + drop(last); + return Ok(rejected_head(connection, Error::Malformed)); + } if let Some((ahead, range)) = connection.read_ahead.take() { if range.len() > buffer.bytes.len() { return Ok(rejected_head(connection, Error::HeadTooLarge)); @@ -697,6 +736,11 @@ impl HttpIo { buffer.bytes[..end].zeroize(); if used > end { connection.read_ahead = Some((buffer, end..used)); + // A later head starts in the last chunk; keep its + // descriptors for that head. + connection.ahead_descriptors = std::mem::take(&mut last); + } else if let Some(sink) = &ancillary { + sink.borrow_mut().absorb(std::mem::take(&mut last)); } let head = connection.state_mut().admit(head)?; // Admission may remove authentication fields, but it must not @@ -726,15 +770,17 @@ impl HttpIo { .reactor .recv_msg(connection.socket(), range, connection, scope) .await?; + // The previous chunk did not complete this head, so its + // descriptors belong to it. let mut sink = sink.borrow_mut(); - sink.truncated |= received.truncated; + sink.absorb(std::mem::take(&mut last)); + last.truncated = received.truncated; for descriptor in received.descriptors { - if sink.descriptors.len() < MAX_HEAD_DESCRIPTORS { - sink.descriptors.push(descriptor); + if last.descriptors.len() < MAX_HEAD_DESCRIPTORS { + last.descriptors.push(descriptor); } else { - // Over the aggregate cap: close now and reject the head. drop(descriptor); - sink.truncated = true; + last.truncated = true; } } Completion { @@ -1112,6 +1158,7 @@ impl ConnectionLease { pool, reusable: false, read_ahead: None, + ahead_descriptors: ReceivedDescriptors::default(), rx_remaining: None, tx_remaining: None, request_is_head: false, @@ -1160,6 +1207,12 @@ impl ConnectionLease { { return Err(Error::Malformed.into()); } + if self.ahead_descriptors.has_any() { + // Carried descriptors without the bytes of their head: close them. + self.ahead_descriptors = ReceivedDescriptors::default(); + self.poison(); + return Err(Error::Malformed.into()); + } self.reusable = false; self.rx_remaining = None; self.tx_remaining = None; @@ -2108,6 +2161,7 @@ fn rejected_head( ) -> HeadCompletion> { connection.poison(); connection.read_ahead = None; + connection.ahead_descriptors = ReceivedDescriptors::default(); connection.rx_remaining = None; connection.request_is_head = false; HeadCompletion { diff --git a/cmd/racer-dataplane/http/tests/exchanges.rs b/cmd/racer-dataplane/http/tests/exchanges.rs index 153c52b96..3d1f430e4 100644 --- a/cmd/racer-dataplane/http/tests/exchanges.rs +++ b/cmd/racer-dataplane/http/tests/exchanges.rs @@ -809,3 +809,89 @@ fn request_head_descriptors_are_capped_across_fragments() { let mut byte = [0]; assert_eq!(read.read(&mut byte).unwrap(), 0); } + +/// Open a pipe; return its read end and an owned write end. +fn descriptor_probe() -> (std::fs::File, std::os::fd::OwnedFd) { + use std::os::fd::FromRawFd; + let mut fds = [0; 2]; + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + unsafe { + ( + std::fs::File::from_raw_fd(fds[0]), + std::os::fd::OwnedFd::from_raw_fd(fds[1]), + ) + } +} + +/// Receive two pipelined heads; the descriptor goes only with the second. +fn pipelined_heads(second: &[&[u8]]) { + use std::io::{Read, Write}; + use std::os::fd::AsRawFd; + let io = io(); + let (mut client, server) = UnixStream::pair().unwrap(); + let server = ConnectionLease::from_reserved(server.into(), (), ()).unwrap(); + let (mut read, write) = descriptor_probe(); + client.write_all(b"HEAD /a HTTP/1.1\r\n\r\n").unwrap(); + send_with_fds(&client, second[0], &[write.as_raw_fd()]); + drop(write); + for part in &second[1..] { + client.write_all(part).unwrap(); + } + let first = drive( + &io, + io.receive_request_head_with_descriptors(server, &RequestScope, 4096), + ) + .unwrap(); + let head = first.value.unwrap(); + assert!(head.descriptors.descriptors.is_empty()); + assert!(!head.descriptors.truncated); + let connection = first.connection; + let second = drive( + &io, + io.receive_request_head_with_descriptors(connection, &RequestScope, 4096), + ) + .unwrap(); + let head = second.value.unwrap(); + assert!(matches!(head.head.start, StartLine::Request { .. })); + assert_eq!(head.descriptors.descriptors.len(), 1); + assert!(!head.descriptors.truncated); + drop(head); + drop(second.connection); + let mut byte = [0]; + assert_eq!(read.read(&mut byte).unwrap(), 0); +} + +/// A plain head and a descriptor-bearing head read by one `recvmsg`. +#[test] +fn pipelined_head_descriptors_attach_to_their_own_head() { + pipelined_heads(&[b"POST /b HTTP/1.1\r\nContent-Length: 0\r\n\r\n"]); +} + +/// The descriptor-bearing skb carries only part of the second head. +#[test] +fn pipelined_fragmented_head_keeps_its_descriptors() { + pipelined_heads(&[b"POST /b HTTP/1.1\r\n", b"Content-Length: 0\r\n\r\n"]); +} + +/// Descriptors carried with read-ahead close when the connection drops. +#[test] +fn carried_head_descriptors_close_with_the_connection() { + use std::io::{Read, Write}; + use std::os::fd::AsRawFd; + let io = io(); + let (mut client, server) = UnixStream::pair().unwrap(); + let server = ConnectionLease::from_reserved(server.into(), (), ()).unwrap(); + let (mut read, write) = descriptor_probe(); + client.write_all(b"HEAD /a HTTP/1.1\r\n\r\n").unwrap(); + send_with_fds(&client, b"POST /b HTTP/1.1\r\n", &[write.as_raw_fd()]); + drop(write); + let first = drive( + &io, + io.receive_request_head_with_descriptors(server, &RequestScope, 4096), + ) + .unwrap(); + assert!(first.value.unwrap().descriptors.descriptors.is_empty()); + drop(first.connection); + let mut byte = [0]; + assert_eq!(read.read(&mut byte).unwrap(), 0); +} diff --git a/cmd/racer-dataplane/src/client/tests.rs b/cmd/racer-dataplane/src/client/tests.rs index 3d323b103..7736b48f1 100644 --- a/cmd/racer-dataplane/src/client/tests.rs +++ b/cmd/racer-dataplane/src/client/tests.rs @@ -1854,8 +1854,15 @@ fn client_disconnect_cancels_pending_metadata_before_acquisition_deadline() { assert_eq!(scope.check(), Ok(())); fixture.wait_for_origin(RequestKind::Head, 1); drop(socket); - for _ in 0..64 { + // The hangup completion can arrive late on a loaded host: pump until it + // is observed and the connection is gone, well inside the acquisition + // deadline. + let wait = Instant::now() + Duration::from_secs(1); + while (!scope.cancellation.is_cancelled() || fixture.listeners.active_connections() != 0) + && Instant::now() < wait + { fixture.pump(16); + std::thread::yield_now(); } assert!(scope.cancellation.is_cancelled()); assert!(Instant::now() < scope.deadline.0); diff --git a/cmd/racer-dataplane/src/client/tests/direct.rs b/cmd/racer-dataplane/src/client/tests/direct.rs index 155c77e04..8d9c0480a 100644 --- a/cmd/racer-dataplane/src/client/tests/direct.rs +++ b/cmd/racer-dataplane/src/client/tests/direct.rs @@ -511,3 +511,34 @@ fn gpu_read_delivers_pages_through_the_backend_seam() { } mod gpu; + +/// A plain HEAD pipelined ahead of a host-read on one connection: one receive +/// can return both heads with the host-read's descriptor. The descriptor must +/// not be attached to the HEAD, so the HEAD is served. The http layer rejects +/// pipelining at exchange completion, so the connection then closes without +/// serving the host-read and closes its carried descriptor. +#[test] +fn pipelined_head_then_host_read_serves_the_head_and_closes_the_descriptor() { + let fixture = direct_fixture(b"hello".to_vec(), None); + let mut memfd = Memfd::new(); + memfd.file.write_all(&[b'.'; 8]).unwrap(); + let mut socket = fixture.connect(); + socket.write_all(&request("HEAD", "")).unwrap(); + send_with_fds( + &socket, + &direct_request( + "host-read", + "Racer-Destination-Offset: 0\r\nRacer-Destination-Length: 5\r\n", + ), + &[memfd.file.as_raw_fd()], + ); + let output = fixture.receive(&mut socket, true); + let (head, rest) = split(&output); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert!(head.contains("content-length: 5\r\n"), "{head}"); + assert!(rest.is_empty(), "pipelined request is not served"); + assert_eq!(memfd.contents(), b"........"); + settle(&fixture); + assert_eq!(memfd.open_copies(), 1, "dataplane closed its copy"); + assert_no_body_leases(&fixture); +} From aada0b333b95c1ad5010072c232bdb177b06228d Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 08:12:21 +0000 Subject: [PATCH 73/87] refactor(racer): build the derived GCM nonce from the XAES nonce directly --- cmd/racer-dataplane/crypto/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/racer-dataplane/crypto/src/lib.rs b/cmd/racer-dataplane/crypto/src/lib.rs index cb9343894..8570f6241 100644 --- a/cmd/racer-dataplane/crypto/src/lib.rs +++ b/cmd/racer-dataplane/crypto/src/lib.rs @@ -357,8 +357,7 @@ pub fn xaes_derive(key: &[u8; 32], nonce: &[u8; 24]) -> DerivedPageKey { let derived = PageCipher::new(key) .derive_key(nonce) .expect("16-byte halves always convert to AES blocks"); - let mut gcm_nonce = [0; 12]; - gcm_nonce.copy_from_slice(&nonce[12..]); + let gcm_nonce: [u8; 12] = std::array::from_fn(|i| nonce[12 + i]); DerivedPageKey { key: derived, nonce: gcm_nonce, From 2fce90206cad1692eabd7dd6a66dff3a8ee79e30 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 16:23:26 +0000 Subject: [PATCH 74/87] fix(racer): reopen slab descriptors without O_NOFOLLOW for cuFile cuFile rejects descriptors with O_NOFOLLOW (error 5019), and Racer opens every slab file and block device with it, so cuFile was turned off on the first direct read. Reopen the open file through /proc/self/fd before registering it. --- cmd/racer-dataplane/gpu/src/lib.rs | 104 +++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/cmd/racer-dataplane/gpu/src/lib.rs b/cmd/racer-dataplane/gpu/src/lib.rs index 0176fb766..aedb9d65f 100644 --- a/cmd/racer-dataplane/gpu/src/lib.rs +++ b/cmd/racer-dataplane/gpu/src/lib.rs @@ -256,6 +256,7 @@ impl Gpu { pub fn open_file(&self, fd: OwnedFd) -> Result { let shared = &self.shared; shared.check(unsafe { (shared.api.file_open)() })?; + let fd = cufile_descriptor(fd); let raw = create(shared, |out| unsafe { (shared.api.file)(fd.as_raw_fd(), out) })?; @@ -782,6 +783,48 @@ impl Drop for Stream { } } +/// Return a descriptor cuFile accepts for the same open file. +/// +/// cuFile refuses to register a descriptor whose status flags include +/// `O_NOFOLLOW` (`CU_FILE_INVALID_FILE_OPEN_FLAG`, 5019), and Racer opens every +/// slab file and block device with it. Reopening `/proc/self/fd/N` follows the +/// kernel's link to the already-open file, not a path, so dropping the flag +/// here does not reintroduce the symlink race it guards against. The new +/// descriptor keeps the access mode and `O_DIRECT`, and is used only if it +/// names the same file. Otherwise the original is returned unchanged and +/// cuFile reports its own error. +fn cufile_descriptor(fd: OwnedFd) -> OwnedFd { + let raw = fd.as_raw_fd(); + let flags = unsafe { libc::fcntl(raw, libc::F_GETFL) }; + if flags < 0 || flags & libc::O_NOFOLLOW == 0 { + return fd; + } + let path = format!("/proc/self/fd/{raw}\0"); + let reopen = (flags & !(libc::O_NOFOLLOW | libc::O_CREAT | libc::O_EXCL | libc::O_TRUNC)) + | libc::O_CLOEXEC; + let new = unsafe { libc::open(path.as_ptr().cast(), reopen) }; + if new < 0 { + return fd; + } + let new = unsafe { OwnedFd::from_raw_fd(new) }; + match (stat(&fd), stat(&new)) { + (Some(a), Some(b)) + if a.st_dev == b.st_dev && a.st_ino == b.st_ino && a.st_rdev == b.st_rdev => + { + new + } + _ => fd, + } +} + +fn stat(fd: &OwnedFd) -> Option { + let mut out = std::mem::MaybeUninit::::uninit(); + if unsafe { libc::fstat(fd.as_raw_fd(), out.as_mut_ptr()) } != 0 { + return None; + } + Some(unsafe { out.assume_init() }) +} + /// A file registered with cuFile. pub struct GpuFile { shared: Rc, @@ -841,6 +884,67 @@ impl Drop for GpuFile { mod tests { use super::*; + fn open(path: &std::path::Path, flags: i32) -> OwnedFd { + let path = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).unwrap(); + let fd = unsafe { libc::open(path.as_ptr(), flags | libc::O_CLOEXEC) }; + assert!(fd >= 0, "open: {}", std::io::Error::last_os_error()); + unsafe { OwnedFd::from_raw_fd(fd) } + } + + fn status(fd: &OwnedFd) -> i32 { + unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFL) } + } + + fn temp_file(name: &str) -> std::path::PathBuf { + let path = std::env::temp_dir().join(format!("racer-gpu-{}-{name}", std::process::id())); + std::fs::write(&path, b"racer").unwrap(); + path + } + + #[test] + /// cuFile rejects O_NOFOLLOW, so the descriptor is reopened without it + /// and keeps the same file and access mode. + fn cufile_descriptor_drops_nofollow() { + let path = temp_file("nofollow"); + let fd = open(&path, libc::O_RDWR | libc::O_NOFOLLOW); + let before = stat(&fd).unwrap(); + let fd = cufile_descriptor(fd); + let after = stat(&fd).unwrap(); + let flags = status(&fd); + assert_eq!(flags & libc::O_NOFOLLOW, 0); + assert_eq!(flags & libc::O_ACCMODE, libc::O_RDWR); + assert_eq!((before.st_dev, before.st_ino), (after.st_dev, after.st_ino)); + let cloexec = unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFD) }; + assert_ne!(cloexec & libc::FD_CLOEXEC, 0); + std::fs::remove_file(path).unwrap(); + } + + #[test] + /// An unlinked file is still reopened through the kernel's link. + fn cufile_descriptor_follows_open_file_not_path() { + let path = temp_file("unlinked"); + let fd = open(&path, libc::O_RDONLY | libc::O_NOFOLLOW); + let before = stat(&fd).unwrap(); + std::fs::remove_file(&path).unwrap(); + std::fs::write(&path, b"other").unwrap(); + let fd = cufile_descriptor(fd); + let after = stat(&fd).unwrap(); + assert_eq!(status(&fd) & libc::O_NOFOLLOW, 0); + assert_eq!((before.st_dev, before.st_ino), (after.st_dev, after.st_ino)); + std::fs::remove_file(path).unwrap(); + } + + #[test] + /// A descriptor without O_NOFOLLOW is returned as is. + fn cufile_descriptor_keeps_other_descriptors() { + let path = temp_file("plain"); + let fd = open(&path, libc::O_RDONLY); + let raw = fd.as_raw_fd(); + let fd = cufile_descriptor(fd); + assert_eq!(fd.as_raw_fd(), raw); + std::fs::remove_file(path).unwrap(); + } + #[test] /// Without a backend in scope or built in, loading fails cleanly. fn load_without_backend_is_unavailable() { From 70f6c274f4abf3457d389e474e9773868584a2c1 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 16:24:17 +0000 Subject: [PATCH 75/87] fix(racer): turn off cuFile compatibility mode by default Without nvidia-fs or P2PDMA, cuFile's compatibility mode reads through a bounce buffer on the GPU helper thread. Racer's host path does the same work on I/O threads, so prefer it unless CUFILE_ALLOW_COMPAT_MODE is set. --- cmd/racer-dataplane/gpu/native/cuda_private.h | 5 +++++ cmd/racer-dataplane/gpu/native/gpu_cuda.c | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/cmd/racer-dataplane/gpu/native/cuda_private.h b/cmd/racer-dataplane/gpu/native/cuda_private.h index bdf9a89b6..cf85676cd 100644 --- a/cmd/racer-dataplane/gpu/native/cuda_private.h +++ b/cmd/racer-dataplane/gpu/native/cuda_private.h @@ -18,6 +18,7 @@ #ifndef RACER_GPU_CUDA_PRIVATE_H #define RACER_GPU_CUDA_PRIVATE_H +#include #include #include #include @@ -175,6 +176,10 @@ _Static_assert(sizeof(CUfileError_t) == 8, "CUfileError_t layout"); _Static_assert(sizeof(CUfileDescr_t) == 24, "CUfileDescr_t layout"); _Static_assert(offsetof(CUfileDescr_t, handle) == 8, "CUfileDescr_t layout"); +/* cuFileSetParameterBool (cuFile 1.10+); resolved optionally. */ +#define CUFILE_PARAM_PROPERTIES_ALLOW_COMPAT_MODE 1 +typedef CUfileError_t (*cuFileSetParameterBool_fn)(int param, bool value); + #define RACER_CUFILE_FUNCTIONS(X) \ X(cuFileDriverOpen, "cuFileDriverOpen", CUfileError_t, (void)) \ X(cuFileHandleRegister, "cuFileHandleRegister", CUfileError_t, \ diff --git a/cmd/racer-dataplane/gpu/native/gpu_cuda.c b/cmd/racer-dataplane/gpu/native/gpu_cuda.c index b2a75a286..e22f4c234 100644 --- a/cmd/racer-dataplane/gpu/native/gpu_cuda.c +++ b/cmd/racer-dataplane/gpu/native/gpu_cuda.c @@ -643,6 +643,16 @@ int racer_gpu_file_open(void) { goto out; } RACER_CUFILE_FUNCTIONS(X_LOAD) + /* Without nvidia-fs or P2PDMA, cuFile's compatibility mode does a + * POSIX read into a bounce buffer and a copy on this helper thread, + * so every GPU job waits behind the disk. Racer's own host path does + * that work on I/O threads. Turn it off unless the operator chose a + * value with CUFILE_ALLOW_COMPAT_MODE; cuFileDriverOpen then fails + * without GPUDirect Storage and Racer uses its host path. */ + cuFileSetParameterBool_fn set_bool = + (cuFileSetParameterBool_fn)dlsym(table.lib, "cuFileSetParameterBool"); + if (set_bool && !getenv("CUFILE_ALLOW_COMPAT_MODE")) + set_bool(CUFILE_PARAM_PROPERTIES_ALLOW_COMPAT_MODE, false); if (file_error(table.cuFileDriverOpen())) { dlclose(table.lib); rc = RACER_GPU_EUNAVAILABLE; From 64a8fac52bc0b5a1f636fb9a6b84ea5c3530beb6 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 16:25:55 +0000 Subject: [PATCH 76/87] fix(racer): give cuFile the udev database and libnuma cuFile classifies each file's block device through udev and fails to register files it cannot classify (error 5030). Mount the host /run/udev read-only when NVIDIA injection is on, and install libnuma1, which cuFile loads at runtime. --- docs/content/guides/racer-vllm.md | 9 +++++-- images/racer-dataplane/Containerfile | 7 +++-- internal/racer/members/members_test.go | 10 +++++-- internal/racer/workload/workload.go | 33 ++++++++++++++++-------- internal/racer/workload/workload_test.go | 21 +++++++++++++++ 5 files changed, 63 insertions(+), 17 deletions(-) diff --git a/docs/content/guides/racer-vllm.md b/docs/content/guides/racer-vllm.md index ff61ba2ee..12a185193 100644 --- a/docs/content/guides/racer-vllm.md +++ b/docs/content/guides/racer-vllm.md @@ -34,8 +34,13 @@ podman build --build-arg RACER_NATIVE_CUDA=true \ The CUDA image also installs NVIDIA's cuFile library (`libcufile-12-9`) for GPUDirect Storage reads, because the NVIDIA container runtime injects only the -driver libraries. cuFile uses its packaged default configuration; without -GPUDirect Storage support on the host, disk reads fall back to host memory. +driver libraries. cuFile uses its packaged default configuration, except that +Racer turns off cuFile's compatibility mode: without GPUDirect Storage +(`nvidia-fs` or PCIe P2PDMA) on the host, disk reads use Racer's own host +memory path instead of cuFile's slower POSIX fallback. Set +`CUFILE_ALLOW_COMPAT_MODE=true` on the dataplane to keep cuFile's fallback. +cuFile reads the filesystem type from udev, so the dataplane mounts the host's +`/run/udev` read-only. GPU mode is `auto` by default. To make the dataplane fail when it finds no GPU, set it to `true`: diff --git a/images/racer-dataplane/Containerfile b/images/racer-dataplane/Containerfile index 3191578a3..032c96fc6 100644 --- a/images/racer-dataplane/Containerfile +++ b/images/racer-dataplane/Containerfile @@ -106,7 +106,9 @@ ARG RACER_NATIVE_RDMA=true ARG RACER_NATIVE_CUDA=false # cuFile user-space library for GPUDirect Storage. The NVIDIA container runtime # injects driver libraries only, not the toolkit's libcufile.so.0, so the CUDA -# image ships it. The repo matches RUNTIME_IMAGE (ubuntu:noble). +# image ships it. The repo matches RUNTIME_IMAGE (ubuntu:noble). cuFile loads +# libnuma.so.1.0.0 at runtime to place its buffers; it is not a package +# dependency. ARG CUDA_CUFILE_PACKAGE=libcufile-12-9 ARG CUDA_RUNTIME_REPO=ubuntu2404 RUN set -eu; \ @@ -124,7 +126,8 @@ RUN set -eu; \ dpkg -i /tmp/cuda-keyring.deb; \ rm /tmp/cuda-keyring.deb; \ apt-get update; \ - apt-get install -y --no-install-recommends "$CUDA_CUFILE_PACKAGE"; \ + apt-get install -y --no-install-recommends "$CUDA_CUFILE_PACKAGE" libnuma1; \ + test -e /usr/lib/"$(uname -m)"-linux-gnu/libnuma.so.1.0.0; \ apt-get purge -y --auto-remove curl; \ lib=$(dirname "$(find /usr/local -name 'libcufile.so.0' | head -n 1)"); \ test -n "$lib" && test "$lib" != .; \ diff --git a/internal/racer/members/members_test.go b/internal/racer/members/members_test.go index 6475b8855..ea8587877 100644 --- a/internal/racer/members/members_test.go +++ b/internal/racer/members/members_test.go @@ -172,6 +172,11 @@ func assertWorkloadVolumes(t *testing.T, pod corev1.PodSpec, cfg Config) { t.Fatal("private keys share disposable storage") } + wantUdev := &corev1.HostPathVolumeSource{Path: "/run/udev", Type: ptr.To(corev1.HostPathDirectoryOrCreate)} + if !reflect.DeepEqual(volumes["udev"].HostPath, wantUdev) { + t.Fatal("cuFile needs the host udev database") + } + wantDevices := &corev1.HostPathVolumeSource{Path: "/dev", Type: ptr.To(corev1.HostPathDirectory)} if !reflect.DeepEqual(volumes["devices"].HostPath, wantDevices) { t.Fatal("block device discovery requires the existing host /dev directory") @@ -201,9 +206,10 @@ func assertWorkloadMounts(t *testing.T, container corev1.Container) { "sockets": {Name: "sockets", MountPath: "/run/racer"}, "infiniband": {Name: "infiniband", MountPath: "/dev/infiniband", ReadOnly: true}, "devices": {Name: "devices", MountPath: "/host/dev", ReadOnly: true}, + "udev": {Name: "udev", MountPath: "/run/udev", ReadOnly: true}, } if !reflect.DeepEqual(mounts, wantMounts) { - t.Fatal("mounts must retain token, controller trust, private identity, slabs, sockets, RDMA devices and host devices only") + t.Fatal("mounts must retain token, controller trust, private identity, slabs, sockets, RDMA devices, host devices and udev only") } } @@ -252,7 +258,7 @@ func TestWorkloadNativeRDMAAccess(t *testing.T) { Path: "/dev/infiniband", Type: ptr.To(corev1.HostPathDirectoryOrCreate), }}} - if len(pod.Volumes) != 7 { + if len(pod.Volumes) != 8 { t.Fatalf("unexpected volumes: %v", pod.Volumes) } diff --git a/internal/racer/workload/workload.go b/internal/racer/workload/workload.go index b28ed66ec..5a0d99f47 100644 --- a/internal/racer/workload/workload.go +++ b/internal/racer/workload/workload.go @@ -364,12 +364,33 @@ func dataplaneVolumes(c Config) []corev1.Volume { volumes = append(volumes, corev1.Volume{Name: mount.name, VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: mount.path, Type: ptr.To(corev1.HostPathDirectoryOrCreate)}}}) } + // cuFile reads the filesystem type of each slab device from the udev + // database and refuses files it cannot classify. + if c.nvidiaMode() != NVIDIAOff { + volumes = append(volumes, corev1.Volume{Name: "udev", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/udev", Type: ptr.To(corev1.HostPathDirectoryOrCreate)}}}) + } + return volumes } func dataplaneContainer(c Config) corev1.Container { diagnosticsPort := diagnosticsPort(c) + mounts := []corev1.VolumeMount{ + {Name: "token", MountPath: "/var/run/racer-token", ReadOnly: true}, + {Name: "bootstrap", MountPath: "/etc/racer/bootstrap", ReadOnly: true}, + {Name: "identity", MountPath: "/var/lib/racer/identity"}, + {Name: "slabs", MountPath: "/var/lib/racer/slabs"}, + {Name: "sockets", MountPath: "/run/racer"}, + // Protect directory entries, not device I/O or the host + // from this privileged container. + {Name: "infiniband", MountPath: "/dev/infiniband", ReadOnly: true}, + {Name: "devices", MountPath: "/host/dev", ReadOnly: true}, + } + if c.nvidiaMode() != NVIDIAOff { + mounts = append(mounts, corev1.VolumeMount{Name: "udev", MountPath: "/run/udev", ReadOnly: true}) + } + return corev1.Container{ Name: "dataplane", Image: c.DataplaneImage, @@ -395,17 +416,7 @@ func dataplaneContainer(c Config) corev1.Container { // implies escalation and all capabilities; do not claim otherwise. Privileged: ptr.To(true), AllowPrivilegeEscalation: ptr.To(true), ReadOnlyRootFilesystem: ptr.To(true), }, - VolumeMounts: []corev1.VolumeMount{ - {Name: "token", MountPath: "/var/run/racer-token", ReadOnly: true}, - {Name: "bootstrap", MountPath: "/etc/racer/bootstrap", ReadOnly: true}, - {Name: "identity", MountPath: "/var/lib/racer/identity"}, - {Name: "slabs", MountPath: "/var/lib/racer/slabs"}, - {Name: "sockets", MountPath: "/run/racer"}, - // Protect directory entries, not device I/O or the host - // from this privileged container. - {Name: "infiniband", MountPath: "/dev/infiniband", ReadOnly: true}, - {Name: "devices", MountPath: "/host/dev", ReadOnly: true}, - }, + VolumeMounts: mounts, } } diff --git a/internal/racer/workload/workload_test.go b/internal/racer/workload/workload_test.go index 455280406..eccdc7f79 100644 --- a/internal/racer/workload/workload_test.go +++ b/internal/racer/workload/workload_test.go @@ -86,6 +86,27 @@ func TestNVIDIAInjectionModes(t *testing.T) { t.Fatalf("CDI annotation = %q (present %v), want present %v", value, hasCDI, tc.wantCDI) } + // cuFile classifies slab devices through the host udev database. + hasUdevVolume := false + + for _, volume := range template.Spec.Volumes { + if volume.Name == "udev" { + hasUdevVolume = volume.HostPath != nil && volume.HostPath.Path == "/run/udev" + } + } + + hasUdevMount := false + + for _, mount := range container.VolumeMounts { + if mount.Name == "udev" { + hasUdevMount = mount.MountPath == "/run/udev" && mount.ReadOnly + } + } + + if hasUdevVolume != tc.wantEnv || hasUdevMount != tc.wantEnv { + t.Fatalf("udev volume/mount = %v/%v, want %v", hasUdevVolume, hasUdevMount, tc.wantEnv) + } + // GPU libraries must never come from a GPU resource request. for _, list := range []corev1.ResourceList{container.Resources.Requests, container.Resources.Limits} { if _, ok := list["nvidia.com/gpu"]; ok { From 7730f3e69d6e082837e85d8289f84522926c4628 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 16:32:51 +0000 Subject: [PATCH 77/87] feat(racer): count GPUDirect disk reads --- .../src/client/tests/direct/gpu.rs | 5 +++ cmd/racer-dataplane/src/read/fill.rs | 6 +++- cmd/racer-dataplane/src/telemetry.rs | 2 ++ cmd/racer-dataplane/src/test_support.rs | 34 ++++++++++++------- docs/content/reference/racer.md | 6 ++++ 5 files changed, 39 insertions(+), 14 deletions(-) diff --git a/cmd/racer-dataplane/src/client/tests/direct/gpu.rs b/cmd/racer-dataplane/src/client/tests/direct/gpu.rs index 6e7dd487c..6e030da04 100644 --- a/cmd/racer-dataplane/src/client/tests/direct/gpu.rs +++ b/cmd/racer-dataplane/src/client/tests/direct/gpu.rs @@ -3,6 +3,7 @@ use super::*; use crate::config::GpuMode; use crate::gpu::GpuClient; use crate::gpu::GpuHelper; +use crate::telemetry::Event; use racer_gpu::simulation::Environment; use racer_gpu::simulation::Live; use racer_gpu::simulation::Operation; @@ -433,6 +434,8 @@ fn gpu_read_disk_pages_go_through_cufile_and_gpu_decrypt() { assert_eq!(worker(&fixture).cpu_decrypts(), 0); assert_eq!(counter(&helper.counters().staged), 3); assert_eq!(counter(&helper.counters().decrypted), 3); + assert_eq!(worker(&fixture).metric(Event::GpuDirectDiskRead), 3); + assert!(worker(&fixture).metric(Event::GpuDirectDiskReadBytes) >= length as u64); assert_idle(&fixture, &helper, default_slots()); } @@ -584,6 +587,8 @@ fn gpu_read_falls_back_to_host_ciphertext_when_cufile_fails() { assert_eq!(calls(&helper, mark, Operation::CopyH2d), 1); assert_eq!(calls(&helper, mark, Operation::Decrypt), 1); assert_eq!(counter(&helper.counters().file_failures), 1); + assert_eq!(worker(&fixture).metric(Event::GpuDirectDiskRead), 0); + assert_eq!(worker(&fixture).metric(Event::GpuDirectDiskReadBytes), 0); assert_eq!(worker(&fixture).cpu_decrypts(), 0); assert_idle(&fixture, &helper, default_slots()); } diff --git a/cmd/racer-dataplane/src/read/fill.rs b/cmd/racer-dataplane/src/read/fill.rs index e662928af..450cf90b1 100644 --- a/cmd/racer-dataplane/src/read/fill.rs +++ b/cmd/racer-dataplane/src/read/fill.rs @@ -1144,13 +1144,14 @@ impl Fill { } let aad = crate::security::page_aad(&envelope)?; let key = self.dependencies.crypto.derive_page_key(&envelope)?; + let read_length = record.read_length; let staged = target .helper .stage_file( target.ordinal, fd, record.ciphertext.offset, - record.read_length as u64, + read_length as u64, scope, ) .await; @@ -1178,6 +1179,9 @@ impl Fill { }); sealed.validate_for(page)?; self.metrics.record(Event::DiskHit, 1); + self.metrics.record(Event::GpuDirectDiskRead, 1); + self.metrics + .record(Event::GpuDirectDiskReadBytes, read_length as u64); Ok(Some(sealed)) } diff --git a/cmd/racer-dataplane/src/telemetry.rs b/cmd/racer-dataplane/src/telemetry.rs index 6b2fe9421..e589c5bb5 100644 --- a/cmd/racer-dataplane/src/telemetry.rs +++ b/cmd/racer-dataplane/src/telemetry.rs @@ -3349,6 +3349,8 @@ metrics! { Event, EVENTS, EVENT_COUNT; Self::FillChecksumCount => "racer_fill_checksum_nanoseconds_count", Self::FillChecksumNs => "racer_fill_checksum_nanoseconds_sum", Self::DiagnosticHardware => "racer_diagnostic_hardware_total", + Self::GpuDirectDiskRead => "racer_gpu_direct_disk_reads_total", + Self::GpuDirectDiskReadBytes => "racer_gpu_direct_disk_read_bytes_total", } metrics! { Gauge, GAUGES, GAUGE_COUNT; Self::WarmTrackedJobs => "racer_warm_tracked_jobs", diff --git a/cmd/racer-dataplane/src/test_support.rs b/cmd/racer-dataplane/src/test_support.rs index a9a7fde92..c0eefcbfe 100644 --- a/cmd/racer-dataplane/src/test_support.rs +++ b/cmd/racer-dataplane/src/test_support.rs @@ -607,19 +607,22 @@ impl ReadWorker { buffers.clone(), ); let memory = Rc::new(MemoryCache::new(buffers.clone(), availability.clone())); - let fill = Rc::new(Fill::new(FillDependencies { - memory: memory.clone(), - buffers, - disk, - writer: writer.clone(), - origin: client.clone(), - candidates: candidates.clone(), - flights: Rc::new(Flights::new(admission.clone(), availability.clone())), - crypto: Rc::new(PageCrypto::new(keys, crypto.clone())), - credentials: credentials.clone(), - admission, - metadata_owner: directory.clone(), - })); + let fill = Rc::new( + Fill::new(FillDependencies { + memory: memory.clone(), + buffers, + disk, + writer: writer.clone(), + origin: client.clone(), + candidates: candidates.clone(), + flights: Rc::new(Flights::new(admission.clone(), availability.clone())), + crypto: Rc::new(PageCrypto::new(keys, crypto.clone())), + credentials: credentials.clone(), + admission, + metadata_owner: directory.clone(), + }) + .with_metrics(metrics.clone()), + ); let metadata = Rc::new(MetadataService::new( candidates, client, @@ -679,6 +682,11 @@ impl ReadWorker { .count(crate::telemetry::Event::CryptoDecryptStarted) } + /// This worker's count of `event`. + pub fn metric(&self, event: crate::telemetry::Event) -> u64 { + self.metrics.count(event) + } + /// Open this worker's slab so pages can be written to and read from disk. pub fn open_disk(&self) { std::fs::create_dir_all(Self::disk_root(&self.origin).join("slabs")).unwrap(); diff --git a/docs/content/reference/racer.md b/docs/content/reference/racer.md index 57a3ff8e6..6147092a7 100644 --- a/docs/content/reference/racer.md +++ b/docs/content/reference/racer.md @@ -1036,6 +1036,12 @@ Where the bytes come from decides who decrypts them: pages from peers. The CPU does not decrypt them. - A page on local disk is read straight from the disk into GPU memory with cuFile when it can be. Otherwise the CPU reads it and copies it to the GPU. + This needs GPUDirect Storage on the host (`nvidia-fs`, or PCIe P2PDMA). + `racer_gpu_direct_disk_reads_total` and + `racer_gpu_direct_disk_read_bytes_total` count these reads. After a cuFile + driver or registration failure the dataplane logs + `stage=gpu state=cufile-disabled` and reads through host memory until it + restarts. - With RDMA on, a page from a peer can be written by the peer's NIC straight into GPU memory on this node (GPUDirect RDMA). See below. From 613331fcc04c77d2ff3829d0cedd17d67f95571e Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 17:14:32 +0000 Subject: [PATCH 78/87] fix(racer): do not retry a failed cuFile driver open --- cmd/racer-dataplane/gpu/native/gpu_cuda.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/racer-dataplane/gpu/native/gpu_cuda.c b/cmd/racer-dataplane/gpu/native/gpu_cuda.c index e22f4c234..fcbe1066a 100644 --- a/cmd/racer-dataplane/gpu/native/gpu_cuda.c +++ b/cmd/racer-dataplane/gpu/native/gpu_cuda.c @@ -631,14 +631,21 @@ static int file_error(CUfileError_t e) { return RACER_GPU_EDRIVER; } +/* Set once cuFile fails to load or open. Opening the driver probes every GPU + * for P2PDMA, which can take most of a second, so a failure is not retried. */ +static int cf_failed; + int racer_gpu_file_open(void) { int rc = RACER_GPU_OK; pthread_mutex_lock(&load_lock); - if (!cf.opened) { + if (cf_failed) { + rc = RACER_GPU_EUNAVAILABLE; + } else if (!cf.opened) { struct cufile_table table; memset(&table, 0, sizeof(table)); table.lib = dlopen("libcufile.so.0", RTLD_NOW | RTLD_LOCAL); if (!table.lib) { + cf_failed = 1; rc = RACER_GPU_EUNAVAILABLE; goto out; } @@ -655,6 +662,7 @@ int racer_gpu_file_open(void) { set_bool(CUFILE_PARAM_PROPERTIES_ALLOW_COMPAT_MODE, false); if (file_error(table.cuFileDriverOpen())) { dlclose(table.lib); + cf_failed = 1; rc = RACER_GPU_EUNAVAILABLE; goto out; } @@ -664,6 +672,7 @@ int racer_gpu_file_open(void) { goto out; missing: dlclose(table.lib); + cf_failed = 1; rc = RACER_GPU_EUNAVAILABLE; } out: From 11a50eb6c4956ffc75f7e1707e40010900355fed Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 17:54:34 +0000 Subject: [PATCH 79/87] fix(racer): decrypt cached ciphertext on the GPU for gpu-read The memory cache now keeps only ciphertext, and the range probe decrypts cached or disk ciphertext on the CPU. Sealed reads probe live plaintext only, and acquire_sealed stages cached ciphertext from memory before trying cuFile or a peer, so the GPU decrypts it. --- .../src/client/tests/direct/gpu.rs | 55 +++++++++++++++++-- cmd/racer-dataplane/src/read/fill.rs | 32 +++++++---- cmd/racer-dataplane/src/read/range_stream.rs | 12 +++- 3 files changed, 81 insertions(+), 18 deletions(-) diff --git a/cmd/racer-dataplane/src/client/tests/direct/gpu.rs b/cmd/racer-dataplane/src/client/tests/direct/gpu.rs index 6e030da04..3fd10e79e 100644 --- a/cmd/racer-dataplane/src/client/tests/direct/gpu.rs +++ b/cmd/racer-dataplane/src/client/tests/direct/gpu.rs @@ -336,8 +336,12 @@ fn arrange( assert!(head.starts_with("http/1.1 200"), "{head}"); let mut copies = Vec::new(); for number in 0..pages { - let result = worker.memory().get(&page_id(number)).unwrap().unwrap(); - copies.push(result.copy()); + let copy = worker + .memory() + .ciphertext(&page_id(number)) + .unwrap() + .unwrap(); + copies.push(copy); } worker.memory().remove_cache(&definition().id).unwrap(); for copy in copies { @@ -372,23 +376,64 @@ fn default_slots() -> usize { (crate::config::DEFAULT_GPU_STAGING_BYTES / crate::gpu::SLOT_BYTES) as usize - 2 } +/// The memory cache keeps only ciphertext, so a page read before is +/// decrypted again, on the GPU, from the cached ciphertext. #[test] -fn gpu_read_memory_plaintext_hit_copies_plaintext() { +fn gpu_read_memory_ciphertext_hit_decrypts_on_the_gpu() { let helper = start(GpuMode::Required, 1).unwrap(); let data = body(5000); let fixture = gpu_fixture(data.clone(), &helper); worker(&fixture).retain(true); let _ = gpu_read(&fixture, &helper, None, 5000); + assert!( + worker(&fixture) + .memory() + .get(&page_id(0)) + .unwrap() + .is_none() + ); + assert!( + worker(&fixture) + .memory() + .ciphertext(&page_id(0)) + .unwrap() + .is_some() + ); let mark = trace_len(&helper); let decrypts = worker(&fixture).cpu_decrypts(); + let decrypted = counter(&helper.counters().decrypted); let (head, frames, allocation, _) = gpu_read(&fixture, &helper, Some((10, 99)), 90); assert!(head.starts_with("http/1.1 200"), "{head}"); assert_eq!(frames, [(1, 0, 10, 90), (2, 1, 90, 0)]); assert_eq!(read_allocation(&allocation, 64, 90), &data[10..100]); - assert_eq!(calls(&helper, mark, Operation::Decrypt), 0); + assert_eq!(calls(&helper, mark, Operation::FileRead), 0); assert_eq!(calls(&helper, mark, Operation::CopyH2d), 1); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 1); + assert_eq!(calls(&helper, mark, Operation::CopyD2d), 1); assert_eq!(worker(&fixture).cpu_decrypts(), decrypts); - assert_eq!(counter(&helper.counters().decrypted), 0); + assert_eq!(counter(&helper.counters().decrypted), decrypted + 1); + assert_eq!(worker(&fixture).metric(Event::GpuDirectDiskRead), 0); + assert_idle(&fixture, &helper, default_slots()); +} + +/// A page cached in memory and on disk is staged from memory: cuFile does +/// not read the disk copy. +#[test] +fn gpu_read_prefers_memory_ciphertext_over_cufile() { + let helper = start(GpuMode::Required, 1).unwrap(); + let data = body(5000); + let fixture = gpu_fixture(data.clone(), &helper); + arrange(&fixture, &helper, 1, 5000, true, true); + assert!(worker(&fixture).disk_location(&page_id(0)).is_some()); + let mark = trace_len(&helper); + let (head, frames, allocation, _) = gpu_read(&fixture, &helper, None, 5000); + assert!(head.starts_with("http/1.1 200"), "{head}"); + assert_eq!(frames.len(), 2, "{frames:?}"); + assert_eq!(read_allocation(&allocation, 64, 5000), data); + assert_eq!(calls(&helper, mark, Operation::FileRead), 0); + assert_eq!(calls(&helper, mark, Operation::Decrypt), 1); + assert_eq!(worker(&fixture).cpu_decrypts(), 0); + assert_eq!(worker(&fixture).metric(Event::GpuDirectDiskRead), 0); assert_idle(&fixture, &helper, default_slots()); } diff --git a/cmd/racer-dataplane/src/read/fill.rs b/cmd/racer-dataplane/src/read/fill.rs index 450cf90b1..55ec6b140 100644 --- a/cmd/racer-dataplane/src/read/fill.rs +++ b/cmd/racer-dataplane/src/read/fill.rs @@ -901,8 +901,9 @@ impl Fill { } } - /// Acquire one page for direct GPU delivery: verified plaintext when the - /// CPU has it anyway, otherwise ciphertext for the GPU to verify and + /// Acquire one page for direct GPU delivery: verified plaintext when a + /// reader still holds it on the CPU, otherwise ciphertext (from memory + /// first, then disk, peers or origin) for the GPU to verify and /// decrypt. Network and disk copies keep the usual single-flight sharing, /// memory publication and disk persistence; only the CPU decrypt and the /// plaintext publication are skipped. @@ -924,7 +925,12 @@ impl Fill { .await .map(Fetched::Page); } - if target.helper.cufile() + // The memory cache keeps only ciphertext once readers drop the + // plaintext. A cached copy skips the disk and peer paths and is sealed + // for the GPU from host memory below. + let cached = self.dependencies.memory.unverified(&page)?.is_some(); + if !cached + && target.helper.cufile() && self .dependencies .writer @@ -941,15 +947,17 @@ impl Fill { Err(_) => {} } } - match self - .land_from_peer(&page, membership.clone(), context, scope, budget, target) - .await - { - Ok(Some(fetched)) => return Ok(fetched), - Ok(None) => {} - Err(error @ (Error::Cancelled | Error::DeadlineExceeded)) => return Err(error), - // Any other failure takes the usual path through host memory. - Err(_) => {} + if !cached { + match self + .land_from_peer(&page, membership.clone(), context, scope, budget, target) + .await + { + Ok(Some(fetched)) => return Ok(fetched), + Ok(None) => {} + Err(error @ (Error::Cancelled | Error::DeadlineExceeded)) => return Err(error), + // Any other failure takes the usual path through host memory. + Err(_) => {} + } } let acquired = self .acquire_with_prefetch( diff --git a/cmd/racer-dataplane/src/read/range_stream.rs b/cmd/racer-dataplane/src/read/range_stream.rs index 0e703279a..25a1670b9 100644 --- a/cmd/racer-dataplane/src/read/range_stream.rs +++ b/cmd/racer-dataplane/src/read/range_stream.rs @@ -782,6 +782,12 @@ impl RangeStream { .as_mut() .and_then(|spill| spill.pages.remove(&number.0)); self.slots.push_back(Slot::Probing(match spilled { + // A sealed read probes live plaintext only. Cached or disk + // ciphertext is then fetched sealed, so the GPU rather than + // the CPU decrypts it. + None if sealed => { + Box::pin(async move { directory.cached_page(page, &scope).await }) + } // Holding pages stay with the spill state until the hold // completes, so the hold is released exactly once. None => Box::pin(async move { directory.local_page(page, &scope).await }), @@ -791,7 +797,11 @@ impl RangeStream { .unwrap() .pages .insert(number.0, Spilling::Holding(holding)); - Box::pin(async move { directory.local_page(page, &scope).await }) + if sealed { + Box::pin(async move { directory.cached_page(page, &scope).await }) + } else { + Box::pin(async move { directory.local_page(page, &scope).await }) + } } // A sealed read probes memory only: a spilled page on disk is // then fetched sealed, so the GPU decrypts it. From ac56411ffe5c9045cd5cf9ffdbca58a0ab9d0ea2 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 17:54:34 +0000 Subject: [PATCH 80/87] docs(racer): describe gpu-read with a ciphertext-only memory cache --- designs/racer-gpudirect.md | 13 ++++++++----- docs/content/reference/racer.md | 3 ++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/designs/racer-gpudirect.md b/designs/racer-gpudirect.md index d263ea9c6..e2ca123d3 100644 --- a/designs/racer-gpudirect.md +++ b/designs/racer-gpudirect.md @@ -170,8 +170,10 @@ startup fail instead. For each page of a `gpu-read`: -- If the plaintext is in memory, the dataplane copies it host to GPU. -- Otherwise it fetches the ciphertext and decrypts it on the GPU (below). +- If another reader still holds the page's plaintext, the dataplane copies it + host to GPU. +- Otherwise it takes the ciphertext (the memory cache keeps only ciphertext) + and decrypts it on the GPU (below). ### GPU decryption @@ -189,7 +191,8 @@ dataplane's GPU memory and are wiped after use. Ciphertext reaches GPU staging memory owned by the dataplane in one of three ways: -- **Memory, origin or HTTP peer:** one host-to-GPU copy. +- **Memory, origin or HTTP peer:** one host-to-GPU copy. A page cached in + memory is staged from there even if it is also on disk. - **Local disk:** cuFile reads it straight from NVMe (aligned records only, below). - **RDMA peer:** the NIC writes it into a GPU landing buffer registered with `ibv_reg_dmabuf_mr`. The dma-buf comes from `cuMemGetHandleForAddressRange`. @@ -200,8 +203,8 @@ How the implementation works today, where it adds to or differs from the plan above: - The read path asks for sealed pages only for `gpu-read` - (`RangeStream::set_sealed`, `Fill::acquire_sealed`). Pages that are verified - plaintext in memory, origin fills (the CPU encrypts them, so it has the + (`RangeStream::set_sealed`, `Fill::acquire_sealed`). Pages whose plaintext + another reader still holds, origin fills (the CPU encrypts them, so it has the plaintext), and pages another reader already decrypted are still delivered as plaintext and copied host to GPU. So are pages that a peer selection fans out to the read window, and pages whose key cannot be derived or whose AAD is diff --git a/docs/content/reference/racer.md b/docs/content/reference/racer.md index 6147092a7..86ae07cdf 100644 --- a/docs/content/reference/racer.md +++ b/docs/content/reference/racer.md @@ -1028,7 +1028,8 @@ for the process. At most 256 GPU reads run at once; more get 503. Where the bytes come from decides who decrypts them: -- A page that is already plaintext in memory is copied to the GPU. +- A page whose plaintext another read still holds in memory is copied to the + GPU. The memory cache itself keeps only encrypted pages. - A page fetched from the origin is encrypted on the CPU, so its plaintext is copied to the GPU too. - Other pages reach the GPU still encrypted, and the GPU checks and decrypts From 0bb90614fc3018268a54fdf940b2a7934ce3ea97 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 21:46:55 +0000 Subject: [PATCH 81/87] feat(racer): run cuFile reads off the GPU helper thread Make the simulation thread safe and add GpuFile::detach and FileReader so a prepared cuFile read can run on another thread while the helper keeps decrypting. --- cmd/racer-dataplane/gpu/src/ffi.rs | 38 ++++--- cmd/racer-dataplane/gpu/src/lib.rs | 104 +++++++++++++++++++- cmd/racer-dataplane/gpu/src/simulation.rs | 31 ++++-- cmd/racer-dataplane/gpu/tests/simulation.rs | 62 ++++++++++++ 4 files changed, 208 insertions(+), 27 deletions(-) diff --git a/cmd/racer-dataplane/gpu/src/ffi.rs b/cmd/racer-dataplane/gpu/src/ffi.rs index 000be6e0f..901ec043a 100644 --- a/cmd/racer-dataplane/gpu/src/ffi.rs +++ b/cmd/racer-dataplane/gpu/src/ffi.rs @@ -268,6 +268,25 @@ impl Drop for Api { } } +/// Map a shim result code to an error. `last_driver_error` must be the +/// calling thread's view of the shim, since the driver error is per thread. +pub(crate) fn check(rc: c_int, last_driver_error: unsafe extern "C" fn() -> i32) -> Result<()> { + match rc { + code::OK => Ok(()), + code::EINVAL => Err(Error::InvalidRequest), + code::ERANGE => Err(Error::InvalidRange), + code::EUNAVAILABLE => Err(Error::Unavailable), + code::EDEVICE => Err(Error::WrongDevice), + code::EDRIVER => Err(Error::Driver(unsafe { last_driver_error() })), + code::ENOMEM => Err(Error::OutOfMemory), + code::ENOKERNEL => Err(Error::KernelUnavailable), + code::EIO => Err(Error::Io), + code::EBUSY => Err(Error::Busy), + code::ELEAKED => Err(Error::Leaked), + _ => Err(Error::Driver(rc)), + } +} + /// The loaded backend and the leak counter every object shares. pub(crate) struct Shared { pub(crate) api: Rc, @@ -278,23 +297,10 @@ pub(crate) struct Shared { impl Shared { /// Map a shim result code to an error. pub(crate) fn check(&self, rc: c_int) -> Result<()> { - match rc { - code::OK => Ok(()), - code::EINVAL => Err(Error::InvalidRequest), - code::ERANGE => Err(Error::InvalidRange), - code::EUNAVAILABLE => Err(Error::Unavailable), - code::EDEVICE => Err(Error::WrongDevice), - code::EDRIVER => Err(Error::Driver(unsafe { (self.api.last_driver_error)() })), - code::ENOMEM => Err(Error::OutOfMemory), - code::ENOKERNEL => Err(Error::KernelUnavailable), - code::EIO => Err(Error::Io), - code::EBUSY => Err(Error::Busy), - code::ELEAKED => { - self.leaks.set(self.leaks.get() + 1); - Err(Error::Leaked) - } - _ => Err(Error::Driver(rc)), + if rc == code::ELEAKED { + self.leaks.set(self.leaks.get() + 1); } + check(rc, self.api.last_driver_error) } /// Count a leaked resource and keep the library loaded for good. diff --git a/cmd/racer-dataplane/gpu/src/lib.rs b/cmd/racer-dataplane/gpu/src/lib.rs index aedb9d65f..27fcdf725 100644 --- a/cmd/racer-dataplane/gpu/src/lib.rs +++ b/cmd/racer-dataplane/gpu/src/lib.rs @@ -3,7 +3,10 @@ //! Every CUDA, NVML and cuFile call goes through a small C shim, //! `libracer_gpu_cuda.so.1` (`native/gpu_cuda.c`), loaded with `dlopen` and //! checked with `racer_gpu_abi()`. All types here are `!Send`: build and use -//! them on the dataplane's native helper thread, never on I/O threads. +//! them on the dataplane's native helper thread, never on I/O threads. The +//! one exception is [`FileReader`], which runs cuFile reads that +//! [`GpuFile::detach`] prepared on other threads, so slow disk reads do not +//! hold up the helper. //! //! # Basics //! @@ -271,8 +274,77 @@ impl Gpu { pub fn leaked(&self) -> u64 { self.shared.leaks.get() } + + /// A handle that runs [`FileRead`]s on any thread. It does not keep the + /// backend loaded: drop every reader thread's work before this [`Gpu`] + /// and the objects the reads name. + pub fn file_reader(&self) -> FileReader { + FileReader { + file_read: self.shared.api.file_read, + last_driver_error: self.shared.api.last_driver_error, + #[cfg(any(test, feature = "simulation"))] + simulation: simulation::current(), + } + } +} + +/// Runs cuFile reads off the helper thread. Cheap to clone. +#[derive(Clone)] +pub struct FileReader { + file_read: unsafe extern "C" fn(*mut c_void, *mut c_void, u64, u64, u64) -> i64, + + last_driver_error: unsafe extern "C" fn() -> i32, + + /// The simulation the backend was loaded in, entered on reader threads. + #[cfg(any(test, feature = "simulation"))] + simulation: Option, +} + +impl FileReader { + /// Run `read` and wait for it. Errors are those of [`GpuFile::read`]. + pub fn read(&self, read: FileRead) -> Result<()> { + #[cfg(any(test, feature = "simulation"))] + let _environment = match (&self.simulation, simulation::current()) { + (Some(simulation), None) => Some(simulation.enter()), + _ => None, + }; + let n = unsafe { + (self.file_read)( + read.file.as_ptr(), + read.staging.as_ptr(), + read.offset, + read.file_offset, + read.length, + ) + }; + if n < 0 { + return ffi::check(n as i32, self.last_driver_error); + } + if n as u64 != read.length { + return Err(Error::Io); + } + Ok(()) + } +} + +/// A cuFile read made by [`GpuFile::detach`], to run with [`FileReader::read`]. +#[derive(Debug)] +pub struct FileRead { + file: NonNull, + + staging: NonNull, + + offset: u64, + + file_offset: u64, + + length: u64, } +// SAFETY: the shim's cuFile read is thread safe. `GpuFile::detach` makes the +// caller keep the file and staging buffer alive until the read is done. +unsafe impl Send for FileRead {} + /// A retained primary context. Clones share it; it is released when the last /// clone and every object made from it are gone. #[derive(Clone)] @@ -868,6 +940,36 @@ impl GpuFile { } Ok(()) } + + /// Prepare a read of `length` bytes at `file_offset` into staging at + /// `offset`, to run on another thread with [`FileReader::read`]. Unlike + /// [`GpuFile::read`], streams may use other parts of the staging buffer + /// meanwhile. + /// + /// # Safety + /// + /// Until the read returns, the caller must keep this file, `staging` and + /// the [`Gpu`] alive, and must not let anything else touch + /// `[offset, offset + length)` of `staging`. + pub unsafe fn detach( + &self, + staging: &StagingBuffer, + offset: u64, + file_offset: u64, + length: u64, + ) -> Result { + let end = offset.checked_add(length).ok_or(Error::InvalidRange)?; + if length == 0 || end > staging.size() { + return Err(Error::InvalidRange); + } + Ok(FileRead { + file: self.raw, + staging: staging.0.raw, + offset, + file_offset, + length, + }) + } } impl Drop for GpuFile { diff --git a/cmd/racer-dataplane/gpu/src/simulation.rs b/cmd/racer-dataplane/gpu/src/simulation.rs index cbdb102f7..d35b2996a 100644 --- a/cmd/racer-dataplane/gpu/src/simulation.rs +++ b/cmd/racer-dataplane/gpu/src/simulation.rs @@ -30,6 +30,7 @@ use std::{ unix::fs::{FileExt, MetadataExt}, }, rc::Rc, + sync::{Arc, Mutex, MutexGuard}, }; /// Granularity imports must be a multiple of. Real GPUs use 2 MiB; smaller @@ -130,7 +131,7 @@ pub struct Live { /// A simulated set of GPUs, shared by clones. Deterministic. #[derive(Clone)] pub struct Simulation { - world: Rc>, + world: Arc>, } /// Scope guard from [`Simulation::enter`]. Restores the previous one on drop. @@ -175,6 +176,9 @@ struct Mapped { length: usize, } +// SAFETY: the mapping is owned and only reached through the world's lock. +unsafe impl Send for Mapped {} + impl Mapped { /// A zeroed shared mapping of `length` bytes, or None. fn new(length: usize) -> Option { @@ -269,10 +273,18 @@ struct World { } impl Simulation { + /// The world, locked. A panic on another thread does not poison it for + /// the rest of a test. + fn world(&self) -> MutexGuard<'_, World> { + self.world + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + /// GPUs that discovery will report, in ordinal order. pub fn new(devices: Vec) -> Self { Self { - world: Rc::new(RefCell::new(World { + world: Arc::new(Mutex::new(World { devices, next: 0x1000, ..World::default() @@ -294,8 +306,7 @@ impl Simulation { let file = unsafe { File::from_raw_fd(fd) }; file.set_len(size).expect("size memfd"); let meta = file.metadata().expect("stat memfd"); - self.world - .borrow_mut() + self.world() .allocations .insert((meta.dev(), meta.ino()), ordinal); file.into() @@ -304,22 +315,22 @@ impl Simulation { /// Fail the next call to `operation` with `code` (a `gpu_cuda.h` code; /// `-5` is a driver error). Nothing changes. pub fn fault(&self, operation: Operation, code: c_int) { - self.world.borrow_mut().faults.push_back((operation, code)); + self.world().faults.push_back((operation, code)); } /// Behave like a shim built without decrypt kernels. pub fn disable_kernels(&self) { - self.world.borrow_mut().no_kernels = true; + self.world().no_kernels = true; } /// Every call so far. pub fn trace(&self) -> Vec { - self.world.borrow().trace.clone() + self.world().trace.clone() } /// Live resources. pub fn live(&self) -> Live { - let world = self.world.borrow(); + let world = self.world(); let (mut imports, mut staging) = (0, 0); for buffer in world.buffers.values() { match buffer.memory { @@ -350,7 +361,7 @@ fn with + Copy + From>( let Some(simulation) = current() else { return R::from(code::EUNAVAILABLE); }; - let mut world = simulation.world.borrow_mut(); + let mut world = simulation.world(); let fault = world .faults .iter() @@ -391,7 +402,7 @@ fn span(size: u64, offset: u64, length: u64) -> bool { } unsafe extern "C" fn last_driver_error() -> i32 { - current().map_or(0, |s| s.world.borrow().last_error) + current().map_or(0, |s| s.world().last_error) } unsafe extern "C" fn init() -> c_int { diff --git a/cmd/racer-dataplane/gpu/tests/simulation.rs b/cmd/racer-dataplane/gpu/tests/simulation.rs index b406b24cb..56fd76ac8 100644 --- a/cmd/racer-dataplane/gpu/tests/simulation.rs +++ b/cmd/racer-dataplane/gpu/tests/simulation.rs @@ -396,6 +396,68 @@ fn cufile_reads_into_staging() { assert_eq!(gpu.leaked(), 0); } +#[test] +/// Detached cuFile reads run on other threads, while a stream uses the same +/// staging buffer, and report errors like a direct read. +fn detached_cufile_reads_run_on_other_threads() { + let sim = simulation(); + let _env = sim.enter(); + let gpu = Gpu::load().unwrap(); + let ctx = gpu.context(0).unwrap(); + let staging = ctx.staging(8192).unwrap(); + staging.register_cufile().unwrap(); + let data: Vec = (0..10000u32).map(|i| (i * 7) as u8).collect(); + let file = gpu.open_file(file_with(&data)).unwrap(); + let reader = gpu.file_reader(); + + // A queued copy into the first page does not block a read of the second. + let stream = ctx.stream().unwrap(); + stream + .copy_from_host(Destination::Staging(&staging), 0, [1]) + .unwrap(); + let reads = [(4096, 0, 1000), (6000, 5000, 2000)].map(|(offset, file_offset, length)| { + unsafe { file.detach(&staging, offset, file_offset, length) }.unwrap() + }); + let threads = reads.map(|read| { + let reader = reader.clone(); + std::thread::spawn(move || reader.read(read)) + }); + for thread in threads { + thread.join().unwrap().unwrap(); + } + stream.synchronize().unwrap(); + let mut out = vec![0; 1000]; + stream.read_staging(&staging, 4096, &mut out).unwrap(); + assert_eq!(out, &data[..1000]); + let mut out = vec![0; 2000]; + stream.read_staging(&staging, 6000, &mut out).unwrap(); + assert_eq!(out, &data[5000..7000]); + + // Bounds are checked up front; I/O and driver errors come from the read. + assert_eq!( + unsafe { file.detach(&staging, 8000, 0, 200) }.err(), + Some(Error::InvalidRange) + ); + assert_eq!( + unsafe { file.detach(&staging, 0, 0, 0) }.err(), + Some(Error::InvalidRange) + ); + let short = unsafe { file.detach(&staging, 0, 9990, 100) }.unwrap(); + let thread = std::thread::spawn({ + let reader = reader.clone(); + move || reader.read(short) + }); + assert_eq!(thread.join().unwrap(), Err(Error::Io)); + sim.fault(Operation::FileRead, -5); + let faulted = unsafe { file.detach(&staging, 0, 0, 100) }.unwrap(); + let thread = std::thread::spawn(move || reader.read(faulted)); + assert!(matches!(thread.join().unwrap(), Err(Error::Driver(_)))); + drop(file); + drop(staging); + assert_eq!(sim.live().files, 0); + assert_eq!(gpu.leaked(), 0); +} + #[test] /// dma-buf export checks page alignment and bounds. fn dma_buf_export_checks_alignment() { From 7a543eb89f4c7720c6df12b8f8e6c61c3dd9ede1 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 21:52:43 +0000 Subject: [PATCH 82/87] perf(racer): read slabs into GPU staging on four reader threads The GPU helper thread ran each cuFile read synchronously, so reads and decryption took turns and one read at a time left the drive mostly idle. Reads now run on four reader threads. The helper keeps file registration and frees a handle only after its last read finishes. --- cmd/racer-dataplane/src/gpu.rs | 355 +++++++++++++++++++++++++++++---- 1 file changed, 318 insertions(+), 37 deletions(-) diff --git a/cmd/racer-dataplane/src/gpu.rs b/cmd/racer-dataplane/src/gpu.rs index 70b700ae0..530a1c621 100644 --- a/cmd/racer-dataplane/src/gpu.rs +++ b/cmd/racer-dataplane/src/gpu.rs @@ -99,6 +99,10 @@ const PLAINTEXT_SCRATCH: u64 = SLOT_BYTES; /// Registered cuFile handles kept per process; more are reopened on demand. const MAX_FILES: usize = 64; +/// Threads that run cuFile reads. One synchronous read keeps an NVMe drive +/// far from busy, and reads on the helper thread would stall decryption. +const FILE_READERS: usize = 4; + /// A shared ciphertext page the GPU stream holds until its copy completes. struct HostCiphertext(crate::memory::CiphertextPage); @@ -806,6 +810,64 @@ struct HelperState { slots: u64, } +/// A registered cuFile handle and the reads using it. +struct OpenFile { + file: racer_gpu::GpuFile, + /// Reads queued or running on a reader thread. + reads: usize, + /// No longer in [`Helper::files`]: free it once `reads` is zero. + retired: bool, +} + +/// A cuFile read for a reader thread. +struct FileJob { + read: racer_gpu::FileRead, + file: u64, + staged: StagedCiphertext, + reply: oneshot::Sender>, +} + +/// A finished cuFile read. +struct FileDone { + file: u64, + failed: bool, +} + +/// Run cuFile reads until the helper stops sending them. Reports each read +/// to the helper before answering, so a later stage sees a failed handle gone. +fn reader_main( + reader: racer_gpu::FileReader, + jobs: Arc>>, + done: mpsc::Sender, + counters: Arc, +) { + loop { + let job = match jobs.lock() { + Ok(jobs) => jobs.recv(), + Err(_) => return, + }; + let Ok(job) = job else { + return; + }; + let result = reader.read(job.read); + let failed = result.is_err(); + if let Err(error) = &result { + counters.file_failures.fetch_add(1, Ordering::Relaxed); + eprintln!("racer-dataplane: stage=gpu state=cufile-read-failed reason={error}"); + } else { + counters.staged.fetch_add(1, Ordering::Relaxed); + } + let _ = done.send(FileDone { + file: job.file, + failed, + }); + // A dropped receiver drops the slot, which frees it. + let _ = job + .reply + .send(result.map(|()| job.staged).map_err(|_| Error::Unavailable)); + } +} + /// State owned by the helper thread. struct Helper { gpu: racer_gpu::Gpu, @@ -814,7 +876,15 @@ struct Helper { staging: HashMap, /// Registered files by (device, inode). The slab descriptors are few and /// long lived, so one registration serves many reads. - files: HashMap<(u64, u64), racer_gpu::GpuFile>, + files: HashMap<(u64, u64), u64>, + /// Registered files by id, including retired ones that reads still use. + open: HashMap, + next_file: u64, + /// Reads for the reader threads. `None` once they are told to stop. + reads: Option>, + /// Reads the reader threads finished, so their files can be released. + done: mpsc::Receiver, + readers: Vec>, requests: HashMap, /// RDMA landing buffers by GPU. Never freed: a NIC may hold a /// registration over them. @@ -833,12 +903,36 @@ fn helper_main( let mut helper = match discover(loader) { Ok((gpu, contexts, usable)) => { let decrypt = contexts.iter().map(probe_decrypt).collect(); + let (reads, queue) = mpsc::channel(); + let queue = Arc::new(Mutex::new(queue)); + let (finished, done) = mpsc::channel(); + let mut readers = Vec::new(); + for _ in 0..FILE_READERS { + let (reader, queue, finished, counters) = ( + gpu.file_reader(), + queue.clone(), + finished.clone(), + state.counters.clone(), + ); + // Without readers, stages fail and pages go through the host. + if let Ok(thread) = std::thread::Builder::new() + .name("racer-gpu-read".into()) + .spawn(move || reader_main(reader, queue, finished, counters)) + { + readers.push(thread); + } + } let _ = ready.send(Ok((usable, decrypt))); Helper { gpu, contexts, staging: HashMap::new(), files: HashMap::new(), + open: HashMap::new(), + next_file: 0, + reads: (!readers.is_empty()).then_some(reads), + done, + readers, requests: HashMap::new(), landing: HashMap::new(), state, @@ -851,9 +945,16 @@ fn helper_main( }; drop(ready); while let Ok(job) = jobs.recv() { + helper.finish_reads(); helper.run(job); } - // Every sender is gone: release what is left, stream first. + // Every sender is gone: let queued reads finish, then release their files. + helper.reads = None; + for reader in std::mem::take(&mut helper.readers) { + let _ = reader.join(); + } + helper.finish_reads(); + // Release what is left, stream first. let ids: Vec = helper.requests.keys().copied().collect(); for id in ids { let _ = helper.release(id); @@ -983,10 +1084,12 @@ impl Helper { length, reply, } => { - let result = self.stage(&staged, fd, file_offset, length); + let result = self.stage(staged, fd, file_offset, length, reply); self.note_leaks(); // A dropped receiver drops the slot, which frees it. - let _ = reply.send(result.map(|()| staged)); + if let Err((error, reply)) = result { + let _ = reply.send(Err(error)); + } } Job::Write { id, @@ -1104,55 +1207,135 @@ impl Helper { self.staging.get(&ordinal).ok_or(Error::Internal) } - /// Read a file range into a staged slot with cuFile. + /// Queue a cuFile read of a file range into a staged slot. A reader + /// thread answers `reply`; on an early error it comes back to the caller. fn stage( &mut self, - staged: &StagedCiphertext, + staged: StagedCiphertext, fd: OwnedFd, file_offset: u64, length: u64, - ) -> Result<()> { - let identity = file_identity(&fd)?; - self.staging(staged.ordinal)?; - let staging = self.staging.get(&staged.ordinal).ok_or(Error::Internal)?; - let counters = &self.state.counters; + reply: oneshot::Sender>, + ) -> std::result::Result<(), (Error, oneshot::Sender>)> { + let Some(reads) = self.reads.clone() else { + return Err((Error::Unavailable, reply)); + }; + let identity = match file_identity(&fd) { + Ok(identity) => identity, + Err(error) => return Err((error, reply)), + }; + if let Err(error) = self.staging(staged.ordinal) { + return Err((error, reply)); + } + let Some(staging) = self.staging.get(&staged.ordinal) else { + return Err((Error::Internal, reply)); + }; if let Err(error) = staging.register_cufile() { - counters.file_failures.fetch_add(1, Ordering::Relaxed); + self.state + .counters + .file_failures + .fetch_add(1, Ordering::Relaxed); self.disable_cufile(error); - return Err(Error::Unavailable); + return Err((Error::Unavailable, reply)); } - if !self.files.contains_key(&identity) { - if self.files.len() >= MAX_FILES { - self.files.clear(); - } - match self.gpu.open_file(fd) { - Ok(file) => { - self.files.insert(identity, file); + let id = match self.files.get(&identity) { + Some(id) => *id, + None => { + if self.files.len() >= MAX_FILES { + for id in std::mem::take(&mut self.files).into_values() { + self.retire(id); + } } - Err(error) => { - counters.file_failures.fetch_add(1, Ordering::Relaxed); - self.disable_cufile(error); - return Err(Error::Unavailable); + match self.gpu.open_file(fd) { + Ok(file) => { + let id = self.next_file; + self.next_file += 1; + self.open.insert( + id, + OpenFile { + file, + reads: 0, + retired: false, + }, + ); + self.files.insert(identity, id); + id + } + Err(error) => { + self.state + .counters + .file_failures + .fetch_add(1, Ordering::Relaxed); + self.disable_cufile(error); + return Err((Error::Unavailable, reply)); + } } } + }; + let (Some(open), Some(staging)) = + (self.open.get_mut(&id), self.staging.get(&staged.ordinal)) + else { + return Err((Error::Internal, reply)); + }; + // SAFETY: the file stays in `open` while `reads` is above zero, the + // staging buffer and backend live until the readers are joined, and + // the slot is this job's alone until the read answers. + let read = match unsafe { + open.file + .detach(staging, staged.offset(), file_offset, length) + } { + Ok(read) => read, + Err(error) => return Err((gpu_error(error), reply)), + }; + open.reads += 1; + let job = FileJob { + read, + file: id, + staged, + reply, + }; + if let Err(mpsc::SendError(job)) = reads.send(job) { + open.reads -= 1; + return Err((Error::Unavailable, job.reply)); } - let file = self.files.get(&identity).ok_or(Error::Internal)?; - let staging = self.staging.get(&staged.ordinal).ok_or(Error::Internal)?; - match file.read(staging, staged.offset(), file_offset, length) { - Ok(()) => { - counters.staged.fetch_add(1, Ordering::Relaxed); - Ok(()) + Ok(()) + } + + /// Account for reads the reader threads finished. A failed handle may be + /// stale, so it is registered again next time. + fn finish_reads(&mut self) { + while let Ok(FileDone { file, failed }) = self.done.try_recv() { + if let Some(open) = self.open.get_mut(&file) { + open.reads = open.reads.saturating_sub(1); } - Err(error) => { - counters.file_failures.fetch_add(1, Ordering::Relaxed); - // A failed handle may be stale; register again next time. - self.files.remove(&identity); - eprintln!("racer-dataplane: stage=gpu state=cufile-read-failed reason={error}"); - Err(Error::Unavailable) + if failed { + self.files.retain(|_, id| *id != file); + self.retire(file); + } else { + self.release_file(file); } } } + /// Stop handing out a file; free it once no read uses it. + fn retire(&mut self, id: u64) { + if let Some(open) = self.open.get_mut(&id) { + open.retired = true; + } + self.release_file(id); + } + + /// Free a retired file that no read uses. + fn release_file(&mut self, id: u64) { + if self + .open + .get(&id) + .is_some_and(|open| open.retired && open.reads == 0) + { + self.open.remove(&id); + } + } + /// Stop trying cuFile after a failure that is not about one read. fn disable_cufile(&self, error: racer_gpu::Error) { if self.state.cufile.swap(false, Ordering::Relaxed) { @@ -1305,7 +1488,7 @@ pub(crate) mod testing { } /// A test scope with a few seconds on the real clock. - fn scope() -> RequestScope { + pub(crate) fn scope() -> RequestScope { RequestScope::new( crate::model::RequestId([1; 16]), std::time::Instant::now() + std::time::Duration::from_secs(5), @@ -1553,4 +1736,102 @@ mod tests { ); assert_eq!(helper.counters().landed.load(Ordering::Relaxed), 0); } + + /// A memfd standing in for a slab file. + fn slab(bytes: &[u8]) -> OwnedFd { + use std::os::unix::fs::FileExt; + let fd = unsafe { libc::memfd_create(c"slab".as_ptr(), libc::MFD_CLOEXEC) }; + assert!(fd >= 0); + let file = unsafe { ::from_raw_fd(fd) }; + file.write_all_at(bytes, 0).unwrap(); + file.into() + } + + /// Stage `length` bytes at `offset` of `fd` on GPU 0, blocking. + fn stage( + helper: &GpuHelper, + fd: &OwnedFd, + offset: u64, + length: u64, + ) -> Result { + futures::executor::block_on(helper.stage_file( + 0, + fd.try_clone().unwrap(), + offset, + length, + &scope(), + )) + } + + fn operations(helper: &GpuHelper, operation: racer_gpu::simulation::Operation) -> usize { + helper.run(move || { + simulation() + .trace() + .iter() + .filter(|event| event.operation == operation) + .count() + }) + } + + #[test] + /// Many stages in flight at once share one registration, each gets its + /// own slot, and the slots come back when the answers drop. + fn concurrent_stages_share_one_file_registration() { + use racer_gpu::simulation::Operation; + let helper = helper(&[None]); + let bytes: Vec = (0..(16 * 4096)).map(|i| (i % 253) as u8).collect(); + let fd = slab(&bytes); + let free = helper.free_slots(0); + let staged = futures::executor::block_on(futures::future::join_all((0..8).map(|i| { + let fd = fd.try_clone().unwrap(); + let scope = scope(); + let helper = helper.clone(); + async move { helper.stage_file(0, fd, i * 4096, 4096, &scope).await } + }))); + let staged: Vec<_> = staged.into_iter().map(Result::unwrap).collect(); + let mut slots: Vec<_> = staged.iter().map(|staged| staged.offset()).collect(); + slots.sort_unstable(); + slots.dedup(); + assert_eq!(slots.len(), 8); + assert_eq!(helper.free_slots(0), free - 8); + assert_eq!(helper.counters().staged.load(Ordering::Relaxed), 8); + assert_eq!(operations(&helper, Operation::FileRead), 8); + assert_eq!(operations(&helper, Operation::File), 1); + drop(staged); + assert_eq!(helper.free_slots(0), free); + assert_eq!(helper.run(|| simulation().live().files), 1); + } + + #[test] + /// A failed read frees its slot and the file is registered again for the + /// next read; cuFile stays on. + fn failed_read_registers_the_file_again() { + use racer_gpu::simulation::Operation; + let helper = helper(&[None]); + let fd = slab(&[7; 8192]); + let free = helper.free_slots(0); + stage(&helper, &fd, 0, 4096).unwrap(); + helper.run(|| simulation().fault(Operation::FileRead, -5)); + assert_eq!(stage(&helper, &fd, 0, 4096).err(), Some(Error::Unavailable)); + assert_eq!(helper.free_slots(0), free); + assert_eq!(helper.counters().file_failures.load(Ordering::Relaxed), 1); + assert!(helper.cufile()); + stage(&helper, &fd, 4096, 4096).unwrap(); + assert_eq!(operations(&helper, Operation::File), 2); + assert_eq!(operations(&helper, Operation::FileFree), 1); + assert_eq!(helper.run(|| simulation().live().files), 1); + assert_eq!(helper.counters().staged.load(Ordering::Relaxed), 2); + } + + #[test] + /// A short file read fails cleanly, as it would on a truncated slab. + fn short_read_fails_the_stage() { + let helper = helper(&[None]); + let fd = slab(&[1; 4096]); + assert_eq!( + stage(&helper, &fd, 2048, 4096).err(), + Some(Error::Unavailable) + ); + assert_eq!(helper.counters().file_failures.load(Ordering::Relaxed), 1); + } } From ee7161228545c027577ae0ec2561850af391d21d Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 22:29:43 +0000 Subject: [PATCH 83/87] perf(racer): open the cuFile driver when the GPU helper starts Opening the driver probes every GPU and took about 0.8s on an eight-GPU B200 node. It ran on the first sealed page, so every first model load paid for it. Open it on a separate thread at startup instead. A failed open turns cuFile off, as a failed stage does. --- cmd/racer-dataplane/gpu/src/lib.rs | 28 +++++++++--- cmd/racer-dataplane/src/gpu.rs | 72 ++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/cmd/racer-dataplane/gpu/src/lib.rs b/cmd/racer-dataplane/gpu/src/lib.rs index 27fcdf725..39b3e9a54 100644 --- a/cmd/racer-dataplane/gpu/src/lib.rs +++ b/cmd/racer-dataplane/gpu/src/lib.rs @@ -47,7 +47,7 @@ pub use ffi::{MAX_AAD, MAX_DECRYPT, flag}; use ffi::{Out, RawDecrypt, RawDevice, RawVerdict, Shared}; use std::{ cell::{Cell, RefCell}, - ffi::c_void, + ffi::{c_int, c_void}, ops::Range, os::fd::{AsRawFd, FromRawFd, OwnedFd}, ptr::NonNull, @@ -280,6 +280,7 @@ impl Gpu { /// and the objects the reads name. pub fn file_reader(&self) -> FileReader { FileReader { + file_open: self.shared.api.file_open, file_read: self.shared.api.file_read, last_driver_error: self.shared.api.last_driver_error, #[cfg(any(test, feature = "simulation"))] @@ -291,6 +292,8 @@ impl Gpu { /// Runs cuFile reads off the helper thread. Cheap to clone. #[derive(Clone)] pub struct FileReader { + file_open: unsafe extern "C" fn() -> c_int, + file_read: unsafe extern "C" fn(*mut c_void, *mut c_void, u64, u64, u64) -> i64, last_driver_error: unsafe extern "C" fn() -> i32, @@ -301,13 +304,19 @@ pub struct FileReader { } impl FileReader { + /// Open the cuFile driver, as [`Gpu::open_file`] does on first use. + /// Opening probes every GPU and can take most of a second, so callers + /// may do it ahead of time on another thread. A failed open is final. + pub fn open_driver(&self) -> Result<()> { + #[cfg(any(test, feature = "simulation"))] + let _environment = self.enter(); + ffi::check(unsafe { (self.file_open)() }, self.last_driver_error) + } + /// Run `read` and wait for it. Errors are those of [`GpuFile::read`]. pub fn read(&self, read: FileRead) -> Result<()> { #[cfg(any(test, feature = "simulation"))] - let _environment = match (&self.simulation, simulation::current()) { - (Some(simulation), None) => Some(simulation.enter()), - _ => None, - }; + let _environment = self.enter(); let n = unsafe { (self.file_read)( read.file.as_ptr(), @@ -325,6 +334,15 @@ impl FileReader { } Ok(()) } + + /// Enter the backend's simulation if this thread is not in one. + #[cfg(any(test, feature = "simulation"))] + fn enter(&self) -> Option { + match (&self.simulation, simulation::current()) { + (Some(simulation), None) => Some(simulation.enter()), + _ => None, + } + } } /// A cuFile read made by [`GpuFile::detach`], to run with [`FileReader::read`]. diff --git a/cmd/racer-dataplane/src/gpu.rs b/cmd/racer-dataplane/src/gpu.rs index 530a1c621..39570e52a 100644 --- a/cmd/racer-dataplane/src/gpu.rs +++ b/cmd/racer-dataplane/src/gpu.rs @@ -885,6 +885,8 @@ struct Helper { /// Reads the reader threads finished, so their files can be released. done: mpsc::Receiver, readers: Vec>, + /// Opens the cuFile driver ahead of the first sealed page. + opener: Option>, requests: HashMap, /// RDMA landing buffers by GPU. Never freed: a NIC may hold a /// registration over them. @@ -922,6 +924,7 @@ fn helper_main( readers.push(thread); } } + let opener = open_cufile(&gpu, &state.cufile); let _ = ready.send(Ok((usable, decrypt))); Helper { gpu, @@ -933,6 +936,7 @@ fn helper_main( reads: (!readers.is_empty()).then_some(reads), done, readers, + opener, requests: HashMap::new(), landing: HashMap::new(), state, @@ -953,6 +957,9 @@ fn helper_main( for reader in std::mem::take(&mut helper.readers) { let _ = reader.join(); } + if let Some(opener) = helper.opener.take() { + let _ = opener.join(); + } helper.finish_reads(); // Release what is left, stream first. let ids: Vec = helper.requests.keys().copied().collect(); @@ -965,6 +972,29 @@ fn helper_main( } } +/// Open the cuFile driver on its own thread. Opening probes every GPU and can +/// take most of a second, so doing it at startup keeps that time out of the +/// first sealed page. A failed open turns cuFile off, as a failed stage does. +fn open_cufile( + gpu: &racer_gpu::Gpu, + cufile: &Arc, +) -> Option> { + if !cufile.load(Ordering::Relaxed) { + return None; + } + let (reader, cufile) = (gpu.file_reader(), cufile.clone()); + std::thread::Builder::new() + .name("racer-gpu-cufile".into()) + .spawn(move || { + if let Err(error) = reader.open_driver() + && cufile.swap(false, Ordering::Relaxed) + { + eprintln!("racer-dataplane: stage=gpu state=cufile-disabled reason={error}"); + } + }) + .ok() +} + /// Queue a zero-length decrypt to learn whether the shim has working decrypt /// kernels for this GPU. It writes nothing. fn probe_decrypt(context: &racer_gpu::Context) -> bool { @@ -1563,6 +1593,14 @@ pub(crate) mod testing { /// A required-mode helper on simulated GPUs with these NUMA nodes. pub(crate) fn helper(numa: &[Option]) -> Arc { + helper_with(numa, |_| {}) + } + + /// A helper whose simulation `setup` prepares before the backend loads. + pub(crate) fn helper_with( + numa: &[Option], + setup: impl FnOnce(&racer_gpu::simulation::Simulation) + Send + 'static, + ) -> Arc { let devices: Vec = numa .iter() .enumerate() @@ -1576,6 +1614,7 @@ pub(crate) mod testing { GpuMode::Required, Box::new(move || { let simulation = racer_gpu::simulation::Simulation::new(devices); + setup(&simulation); let environment = simulation.enter(); SIM.with(|sim| *sim.borrow_mut() = Some((simulation, environment))); racer_gpu::Gpu::load() @@ -1823,6 +1862,39 @@ mod tests { assert_eq!(helper.counters().staged.load(Ordering::Relaxed), 2); } + /// Wait for the helper's background work to reach `done`. + fn eventually(mut done: impl FnMut() -> bool) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !done() { + assert!(std::time::Instant::now() < deadline, "timed out"); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } + + #[test] + /// The cuFile driver opens at startup, before any sealed page asks for it. + fn cufile_driver_opens_at_startup() { + use racer_gpu::simulation::Operation; + let helper = helper(&[None]); + eventually(|| operations(&helper, Operation::FileOpen) == 1); + assert!(helper.cufile()); + let fd = slab(&[3; 4096]); + stage(&helper, &fd, 0, 4096).unwrap(); + assert_eq!(helper.counters().staged.load(Ordering::Relaxed), 1); + } + + #[test] + /// A driver that fails to open at startup turns cuFile off before any + /// sealed page asks for it. + fn cufile_driver_open_failure_turns_cufile_off() { + use racer_gpu::simulation::Operation; + let helper = helper_with(&[None], |simulation| { + simulation.fault(Operation::FileOpen, -5) + }); + eventually(|| !helper.cufile()); + assert_eq!(helper.counters().staged.load(Ordering::Relaxed), 0); + } + #[test] /// A short file read fails cleanly, as it would on a truncated slab. fn short_read_fails_the_stage() { From 5f979a836d43f7f16054bd30243a5b9df8ba4b15 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 22:44:38 +0000 Subject: [PATCH 84/87] perf(racer): end vLLM loader chunks on Racer page boundaries Neighboring chunks used to read the page they share twice. Chunks now end on page boundaries where they can, and a tensor that crosses one is copied out of each chunk into its own memory. --- .../racer-vllm/src/racer_vllm/safetensors.py | 49 +++++++++++++-- .../racer-vllm/src/racer_vllm/weights.py | 54 ++++++++++++++++- .../python/racer-vllm/tests/test_loader.py | 26 ++++++++ .../racer-vllm/tests/test_safetensors.py | 60 +++++++++++++++++++ clients/python/racer/src/racer/__init__.py | 2 + 5 files changed, 183 insertions(+), 8 deletions(-) diff --git a/clients/python/racer-vllm/src/racer_vllm/safetensors.py b/clients/python/racer-vllm/src/racer_vllm/safetensors.py index 1d92fb17b..2574e35a5 100644 --- a/clients/python/racer-vllm/src/racer_vllm/safetensors.py +++ b/clients/python/racer-vllm/src/racer_vllm/safetensors.py @@ -119,28 +119,57 @@ def parse_header(raw: bytes, file_size: int | None = None) -> Header: return Header(data_start, tensors, metadata) +@dataclass(frozen=True) +class Part: + """Bytes [offset, offset + length) of a tensor that spans several chunks.""" + + tensor: TensorInfo + offset: int + length: int + + @property + def last(self) -> bool: + return self.offset + self.length == self.tensor.nbytes + + @dataclass class Chunk: """A contiguous file range read with one request.""" start: int end: int - tensors: list + tensors: list # tensors that lie wholly inside the chunk dedicated: bool = False # larger than the staging buffer + parts: list = field(default_factory=list) # Parts of tensors split across chunks @property def length(self) -> int: return self.end - self.start -def plan_chunks(tensors: list, staging_bytes: int) -> list: +def plan_chunks(tensors: list, staging_bytes: int, page_bytes: int = 0) -> list: """Group sorted tensors into ranges of at most staging_bytes. A tensor bigger than staging_bytes gets its own dedicated chunk. Zero-size tensors join the current chunk and need no bytes. + + With page_bytes no larger than staging_bytes, a chunk that would end in + the middle of a page ends on the page boundary instead, and the tensor that + crosses the boundary is split into Parts read by consecutive chunks. Without + this, both chunks read the boundary page. """ + paged = 0 < page_bytes <= staging_bytes chunks: list = [] cur: Chunk | None = None + limit = 0 + + def open_chunk(start: int) -> Chunk: + nonlocal limit + limit = start + staging_bytes + if paged: + limit -= limit % page_bytes + return Chunk(start, start, []) + for t in tensors: if t.nbytes > staging_bytes: if cur is not None: @@ -148,13 +177,23 @@ def plan_chunks(tensors: list, staging_bytes: int) -> list: cur = None chunks.append(Chunk(t.start, t.end, [t], dedicated=True)) continue - if cur is not None and t.end - cur.start <= staging_bytes: + if cur is not None and t.end > limit and (t.start >= limit or not paged): + chunks.append(cur) + cur = None + if cur is None: + cur = open_chunk(t.start) + if t.end <= limit: cur.tensors.append(t) cur.end = max(cur.end, t.end) continue - if cur is not None: + pos = t.start + while t.end > limit: + cur.parts.append(Part(t, pos - t.start, limit - pos)) + cur.end = pos = limit chunks.append(cur) - cur = Chunk(t.start, t.end, [t]) + cur = open_chunk(pos) + cur.parts.append(Part(t, pos - t.start, t.end - pos)) + cur.end = t.end if cur is not None: chunks.append(cur) return chunks diff --git a/clients/python/racer-vllm/src/racer_vllm/weights.py b/clients/python/racer-vllm/src/racer_vllm/weights.py index dea0823a8..21b1c7ce7 100644 --- a/clients/python/racer-vllm/src/racer_vllm/weights.py +++ b/clients/python/racer-vllm/src/racer_vllm/weights.py @@ -35,6 +35,18 @@ def new_dedicated(self, size: int): def tensor(self, target, rel: int, info: TensorInfo, staged: bool): raise NotImplementedError + def new_joined(self, info: TensorInfo): + """Memory for a tensor that is split across chunks.""" + raise NotImplementedError + + def copy_part(self, joined, offset: int, target, rel: int, length: int) -> None: + """Copy length bytes at rel in target to offset in joined.""" + raise NotImplementedError + + def joined_tensor(self, joined, info: TensorInfo): + """Return the tensor once every part has been copied into joined.""" + raise NotImplementedError + def done(self, target) -> None: """The consumer is finished with every tensor from target.""" @@ -42,7 +54,7 @@ def wait(self, target) -> None: """Block until work queued by the consumer on target has finished.""" -def _file_chunks(client: racer.Client, uri: ModelURI, staging_bytes: int): +def _file_chunks(client: racer.Client, uri: ModelURI, staging_bytes: int, page_bytes: int): for name in weight_files(client, uri): req = uri.request(name) meta = client.stat(req) @@ -50,7 +62,7 @@ def _file_chunks(client: racer.Client, uri: ModelURI, staging_bytes: int): n = header_length(prefix) header = parse_header(client.read(req, 8, n, if_match=meta.etag), meta.size) log.debug("racer: %s has %d tensors", name, len(header.tensors)) - for chunk in plan_chunks(header.tensors, staging_bytes): + for chunk in plan_chunks(header.tensors, staging_bytes, page_bytes): yield req, meta.etag, chunk @@ -60,6 +72,7 @@ def iter_weights( backend: Backend, staging_bytes: int = DEFAULT_STAGING_BYTES, depth: int = DEFAULT_DEPTH, + page_bytes: int = racer.PAGE_SIZE, ) -> Iterator[tuple]: """Yield (name, tensor) for every tensor in the model. @@ -72,11 +85,17 @@ def iter_weights( are never released while queued GPU work may still read them. Dedicated targets are fenced and dropped as soon as the consumer resumes past them, so oversized tensors do not accumulate in GPU memory. + + Chunks end on Racer page boundaries (page_bytes) where they can, so + neighboring chunks do not both read the boundary page. A tensor that + crosses a boundary is copied out of each chunk into memory from + backend.new_joined() and yielded with the chunk that holds its last byte. """ if depth < 1: raise ValueError("depth must be at least 1") free = [backend.new_staging(staging_bytes) for _ in range(depth)] - chunks = _file_chunks(client, uri, staging_bytes) + chunks = _file_chunks(client, uri, staging_bytes, page_bytes) + joined: dict = {} inflight: collections.deque = collections.deque() pool = ThreadPoolExecutor(max_workers=depth, thread_name_prefix="racer-read") @@ -107,6 +126,14 @@ def fill() -> bool: active = target for t in chunk.tensors: yield t.name, backend.tensor(target, t.start - chunk.start, t, not chunk.dedicated) + for part in chunk.parts: + t = part.tensor + if part.offset == 0: + joined[t.name] = backend.new_joined(t) + rel = t.start + part.offset - chunk.start + backend.copy_part(joined[t.name], part.offset, target, rel, part.length) + if part.last: + yield t.name, backend.joined_tensor(joined.pop(t.name), t) active = None backend.done(target) if chunk.dedicated: @@ -150,6 +177,15 @@ def new_dedicated(self, size: int): def tensor(self, target, rel, info, staged): return bytes(target.buf[rel : rel + info.nbytes]) + def new_joined(self, info): + return bytearray(info.nbytes) + + def copy_part(self, joined, offset, target, rel, length): + joined[offset : offset + length] = target.buf[rel : rel + length] + + def joined_tensor(self, joined, info): + return bytes(joined) + TORCH_DTYPES = { "BOOL": "bool", @@ -230,6 +266,18 @@ def tensor(self, target, rel, info, staged): raw = raw.clone() return raw.view(dtype).view(info.shape) + def new_joined(self, info): + return self.torch.empty(info.nbytes, dtype=self.torch.uint8, device=f"cuda:{self.device}") + + def copy_part(self, joined, offset, target, rel, length): + # Queued before done(target), so the staging buffer is not reused + # until the copy has finished. + joined[offset : offset + length].copy_(self._flat[id(target)][rel : rel + length]) + + def joined_tensor(self, joined, info): + dtype = getattr(self.torch, TORCH_DTYPES[info.dtype]) + return joined.view(dtype).view(info.shape) + def done(self, target): # Record on the target device's stream; the calling thread's current # device may differ, and its stream would not cover our copies. diff --git a/clients/python/racer-vllm/tests/test_loader.py b/clients/python/racer-vllm/tests/test_loader.py index 463a38c1f..d06c8bb3c 100644 --- a/clients/python/racer-vllm/tests/test_loader.py +++ b/clients/python/racer-vllm/tests/test_loader.py @@ -105,6 +105,32 @@ def test_iter_weights(server, uri, depth): assert 900 in sizes +@pytest.mark.parametrize("depth", [1, 2, 3]) +def test_iter_weights_reads_each_page_once(server, uri, depth): + setup_model(server, uri) + c = racer.Client(socket_path=server.path, staging_bytes=4096) + b = _FencingBackend() + got = dict(iter_weights(c, uri, b, staging_bytes=400, depth=depth, page_bytes=64)) + assert got == {"w1": b"\1" * 300, "w2": b"\2" * 50, "w3": b"\3" * 900, "w4": b"\0" * 8} + b.assert_fenced() + # Staged weight reads (not the two header reads, not the dedicated w3 read) + # never share a page. + pages = {} + reads = 0 + for s in server.seen: + if s.method == "POST" and "range" in s.headers: + first, last = (int(x) for x in s.headers["range"][len("bytes=") :].split("-")) + if first in (0, 8) or last + 1 - first == 900: + continue + reads += 1 + mine = set(range(first // 64, last // 64 + 1)) + assert not mine & pages.setdefault(s.target, set()), (s.target, first, last) + pages[s.target] |= mine + assert reads >= 2 + + +class _Param: + pass class _Param: pass diff --git a/clients/python/racer-vllm/tests/test_safetensors.py b/clients/python/racer-vllm/tests/test_safetensors.py index 9648e3762..e57988740 100644 --- a/clients/python/racer-vllm/tests/test_safetensors.py +++ b/clients/python/racer-vllm/tests/test_safetensors.py @@ -56,6 +56,66 @@ def test_plan_chunks_split_and_dedicated(): assert all(c.length <= 250 for c in chunks if not c.dedicated) +def _pages(start, end, page): + return set(range(start // page, (end - 1) // page + 1)) if end > start else set() + + +def test_plan_chunks_on_page_boundaries(): + f = make_safetensors([(f"t{i}", "U8", (90,), bytes([i]) * 90) for i in range(8)] + [("z", "U8", (0,), b"")]) + n = header_length(f[:8]) + h = parse_header(f[8 : 8 + n]) + chunks = plan_chunks(h.tensors, 250, 64) + # Every chunk but the last ends on a page boundary, and no two chunks share a page. + assert all(c.end % 64 == 0 for c in chunks[:-1]) + assert all(c.length <= 250 for c in chunks) + seen = set() + for c in chunks: + pages = _pages(c.start, c.end, 64) + assert not pages & seen + seen |= pages + # Each tensor is either whole in one chunk or covered exactly once by its parts. + covered = {t.name: [] for t in h.tensors} + for c in chunks: + for t in c.tensors: + assert c.start <= t.start and t.end <= c.end + covered[t.name].append((0, t.nbytes)) + for p in c.parts: + assert c.start <= p.tensor.start + p.offset and p.tensor.start + p.offset + p.length <= c.end + covered[p.tensor.name].append((p.offset, p.offset + p.length)) + for t in h.tensors: + spans = sorted(covered[t.name]) + assert spans[0][0] == 0 and spans[-1][1] == t.nbytes, t.name + assert all(a[1] == b[0] for a, b in zip(spans, spans[1:])), t.name + assert any(c.parts for c in chunks) + assert [p.last for c in chunks for p in c.parts].count(True) == len({p.tensor.name for c in chunks for p in c.parts}) + + +def test_plan_chunks_splits_across_several_chunks(): + # With staging of just under two pages, each chunk after the first holds + # one page, so a tensor that starts mid-chunk spans three chunks. + f = make_safetensors([("a", "U8", (100,), b"a" * 100), ("b", "U8", (127,), b"b" * 127)]) + n = header_length(f[:8]) + h = parse_header(f[8 : 8 + n]) + chunks = plan_chunks(h.tensors, 127, 64) + parts = [p for c in chunks for p in c.parts if p.tensor.name == "b"] + assert len(parts) >= 3 + assert [(p.offset, p.length) for p in parts][0][0] == 0 + assert all(a.offset + a.length == b.offset for a, b in zip(parts, parts[1:])) + assert sum(p.length for p in parts) == 127 + assert [p.last for p in parts] == [False] * (len(parts) - 1) + [True] + assert all(c.length <= 127 for c in chunks) + + +def test_plan_chunks_ignores_pages_larger_than_staging(): + f = make_safetensors([(f"t{i}", "U8", (100,), bytes([i]) * 100) for i in range(5)]) + n = header_length(f[:8]) + h = parse_header(f[8 : 8 + n]) + assert [[t.name for t in c.tensors] for c in plan_chunks(h.tensors, 250, 1024)] == [ + [t.name for t in c.tensors] for c in plan_chunks(h.tensors, 250) + ] + assert not any(c.parts for c in plan_chunks(h.tensors, 250, 1024)) + + @pytest.mark.parametrize( "header", [ diff --git a/clients/python/racer/src/racer/__init__.py b/clients/python/racer/src/racer/__init__.py index 8fc7cf085..2a338da74 100644 --- a/clients/python/racer/src/racer/__init__.py +++ b/clients/python/racer/src/racer/__init__.py @@ -21,12 +21,14 @@ UnsatisfiableRange, VersionUnavailable, ) +from ._wire import PAGE_SIZE from .host import HostBuffer from .keys import Request, canonical_json, object_key, object_request __all__ = [ "GPU", "HOST", + "PAGE_SIZE", "STREAM", "Client", "HostBuffer", From d0a847285a3f3fbaa7f37369a573ce0abadcbf5e Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 23:11:10 +0000 Subject: [PATCH 85/87] docs(racer): describe cuFile reader threads and the eager driver open --- designs/racer-gpudirect.md | 18 +++++++++++++----- docs/content/reference/racer.md | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/designs/racer-gpudirect.md b/designs/racer-gpudirect.md index e2ca123d3..44fdb546c 100644 --- a/designs/racer-gpudirect.md +++ b/designs/racer-gpudirect.md @@ -157,7 +157,8 @@ A new crate, `cmd/racer-dataplane/gpu`, follows the `verbs` pattern: - A small C shim, `libracer_gpu_cuda.so.1`, wraps the CUDA driver API, NVML and cuFile. Racer loads it with `dlopen` and checks `racer_gpu_abi()`. -- Calls run on the native helper thread, never on I/O threads. +- Calls run on the native helper thread, never on I/O threads. The one + exception is cuFile reads, which run on four reader threads (below). - A simulation backend backs tests without GPUs. It can inject failures. - The Cargo feature is `cuda`. Without it, GPU mode is unavailable. @@ -224,10 +225,17 @@ above: wiped; it is dataplane-owned GPU memory that is never read back to the host. - The other slots hold cuFile reads. The page's owning worker reads the record header through host memory (`StoreReader::direct_read`), holds the segment - lease, and asks the GPU helper to read the ciphertext into a free slot. When - no slot is free, cuFile fails, or the record is v1, unaligned, or on a - simulated descriptor, the worker reads the page through host memory instead. - A cuFile driver or registration failure turns cuFile off until restart. + lease, and asks the GPU helper to read the ciphertext into a free slot. The + helper registers the file with cuFile once and hands the read to one of four + reader threads, so several reads run at once while the helper decrypts. In a + test on one NVMe drive, one thread reached about 5 GiB/s and two or more + reached the drive's limit of about 6 GiB/s. When no slot is free, cuFile + fails, or the record is v1, unaligned, or on a simulated descriptor, the + worker reads the page through host memory instead. +- The helper opens the cuFile driver on another thread at startup. Opening it + probes every GPU and can take most of a second, which would otherwise delay + the first disk read. A cuFile driver or registration failure turns cuFile off + until restart. A failed read only drops that file's registration. - At startup the helper runs a zero-length decrypt on each GPU. A GPU whose shim has no decrypt kernels, or a staging cap smaller than the two scratch slots, gets plaintext pages only. diff --git a/docs/content/reference/racer.md b/docs/content/reference/racer.md index 86ae07cdf..8992ff053 100644 --- a/docs/content/reference/racer.md +++ b/docs/content/reference/racer.md @@ -1024,7 +1024,7 @@ GPU memory. They need: At startup the dataplane logs each GPU it finds: ordinal, UUID, PCI address, NUMA node, compute mode, and whether it can use it. It skips GPUs in `EXCLUSIVE_PROCESS` or `PROHIBITED` mode. One helper thread does all GPU work -for the process. At most 256 GPU reads run at once; more get 503. +for the process, except cuFile disk reads, which run on four reader threads. At most 256 GPU reads run at once; more get 503. Where the bytes come from decides who decrypts them: From f0a94a1ccfe3a7308e9ee81a903bf558318c8680 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Fri, 23 Oct 2026 23:11:10 +0000 Subject: [PATCH 86/87] perf(racer): keep four vLLM loader reads in flight by default With page-aligned chunks and parallel cuFile reads, four staging buffers load a 7B model about 7% faster than two on one NVMe drive, close to the drive's GPUDirect limit. --- clients/python/racer-vllm/README.md | 9 ++++++--- clients/python/racer-vllm/src/racer_vllm/loader.py | 2 +- clients/python/racer-vllm/src/racer_vllm/weights.py | 2 +- clients/python/racer-vllm/tests/test_loader.py | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/clients/python/racer-vllm/README.md b/clients/python/racer-vllm/README.md index d28a76cc8..54340209a 100644 --- a/clients/python/racer-vllm/README.md +++ b/clients/python/racer-vllm/README.md @@ -37,7 +37,7 @@ Then start vLLM on that directory: ``` vllm serve /models/llama-3-8b --load-format racer \ - --model-loader-extra-config '{"staging_bytes": 268435456, "concurrency": 2}' + --model-loader-extra-config '{"staging_bytes": 268435456, "concurrency": 4}' ``` Extra config keys, all optional: @@ -46,7 +46,7 @@ Extra config keys, all optional: |---|---|---| | `uri` | from `racer-model.json` | The `racer://` model location. | | `staging_bytes` | 256 MiB | Size of each GPU staging buffer. | -| `concurrency` | 2 | Staging buffers, and reads in flight. | +| `concurrency` | 4 | Staging buffers, and reads in flight. | | `clone` | true | Copy each tensor out of staging before handing it to vLLM. | | `socket` | none | Client socket path. | | `use_vmm` | true | Allow exportable `cuMemCreate` staging, needed for `gpu-read`. | @@ -56,7 +56,10 @@ Extra config keys, all optional: 1. Find the safetensors files from `model.safetensors.index.json`, or use `model.safetensors`. 2. For each file, read the 8-byte header length and the JSON header. -3. Group tensors into ranges that fit a staging buffer. Read each range into +3. Group tensors into ranges that fit a staging buffer. Each range ends on a + Racer page boundary (16 MiB), so no page is read twice. A tensor that + crosses the boundary is split across two ranges and joined again before vLLM + sees it. Read each range into GPU memory with `gpu-read`. If the dataplane cannot do that, use `host-read` into pinned memory and copy, else the normal stream. 4. Give each tensor (right dtype and shape) to `model.load_weights`. While diff --git a/clients/python/racer-vllm/src/racer_vllm/loader.py b/clients/python/racer-vllm/src/racer_vllm/loader.py index 58a08a558..e578482be 100644 --- a/clients/python/racer-vllm/src/racer_vllm/loader.py +++ b/clients/python/racer-vllm/src/racer_vllm/loader.py @@ -42,7 +42,7 @@ class RacerModelLoader(base): uri racer:////; optional when the model directory has racer-model.json (racer-vllm-fetch) staging_bytes size of each GPU staging buffer (default 256 MiB) - concurrency number of staging buffers and reads in flight (default 2) + concurrency number of staging buffers and reads in flight (default 4) clone copy each tensor out of staging (default true) socket client socket path (else RACER_SOCKET, else the cache) use_vmm allow exportable cuMemCreate staging (default true) diff --git a/clients/python/racer-vllm/src/racer_vllm/weights.py b/clients/python/racer-vllm/src/racer_vllm/weights.py index 21b1c7ce7..5637aeec1 100644 --- a/clients/python/racer-vllm/src/racer_vllm/weights.py +++ b/clients/python/racer-vllm/src/racer_vllm/weights.py @@ -20,7 +20,7 @@ log = logging.getLogger(__name__) DEFAULT_STAGING_BYTES = 256 * 1024 * 1024 -DEFAULT_DEPTH = 2 +DEFAULT_DEPTH = 4 class Backend: diff --git a/clients/python/racer-vllm/tests/test_loader.py b/clients/python/racer-vllm/tests/test_loader.py index d06c8bb3c..0166cbd50 100644 --- a/clients/python/racer-vllm/tests/test_loader.py +++ b/clients/python/racer-vllm/tests/test_loader.py @@ -90,7 +90,7 @@ def test_weight_files_single(server, uri): model.weight_files(c, uri) -@pytest.mark.parametrize("depth", [1, 2, 3]) +@pytest.mark.parametrize("depth", [1, 2, 3, 4]) def test_iter_weights(server, uri, depth): setup_model(server, uri) c = racer.Client(socket_path=server.path, staging_bytes=4096) @@ -105,7 +105,7 @@ def test_iter_weights(server, uri, depth): assert 900 in sizes -@pytest.mark.parametrize("depth", [1, 2, 3]) +@pytest.mark.parametrize("depth", [1, 2, 3, 4]) def test_iter_weights_reads_each_page_once(server, uri, depth): setup_model(server, uri) c = racer.Client(socket_path=server.path, staging_bytes=4096) From 682c723fd5354cb76caa7a7eb5c76f50ae93d6d5 Mon Sep 17 00:00:00 2001 From: Jordan Olshevski Date: Sat, 24 Oct 2026 15:03:33 +0000 Subject: [PATCH 87/87] perf(racer): fill a GPU read's whole window directly A gpu-read now fetches every page of its RACER_RANGE_WINDOW_PAGES window with direct fills instead of only the head and one page ahead. Provider selections deliver verified plaintext, so they cannot use cuFile staging. Direct reads also get at least range_window_pages credits. The defaults are unchanged; the reference and the racer-vllm README describe how to raise the window, GPU staging, and loader reads for fast local disks. --- clients/python/racer-vllm/README.md | 5 +++ cmd/racer-dataplane/src/client/direct.rs | 5 ++- cmd/racer-dataplane/src/read.rs | 8 ++-- cmd/racer-dataplane/src/read/range_stream.rs | 47 ++++++++++++++++++-- docs/content/reference/racer.md | 9 ++++ 5 files changed, 66 insertions(+), 8 deletions(-) diff --git a/clients/python/racer-vllm/README.md b/clients/python/racer-vllm/README.md index 54340209a..501919aea 100644 --- a/clients/python/racer-vllm/README.md +++ b/clients/python/racer-vllm/README.md @@ -51,6 +51,11 @@ Extra config keys, all optional: | `socket` | none | Client socket path. | | `use_vmm` | true | Allow exportable `cuMemCreate` staging, needed for `gpu-read`. | +On fast local disks, more reads in flight help only when the dataplane also +fetches further ahead. Set `concurrency` to 8 together with a larger +`RACER_RANGE_WINDOW_PAGES` and `RACER_GPU_STAGING_BYTES` on the dataplane. See +the GPU reads section of the Racer reference. + ## How it loads 1. Find the safetensors files from `model.safetensors.index.json`, or use diff --git a/cmd/racer-dataplane/src/client/direct.rs b/cmd/racer-dataplane/src/client/direct.rs index e6799ed5a..61c648804 100644 --- a/cmd/racer-dataplane/src/client/direct.rs +++ b/cmd/racer-dataplane/src/client/direct.rs @@ -28,8 +28,9 @@ use std::os::fd::OwnedFd; use std::rc::Rc; use uring_runtime::reactor::descriptor::Descriptor; -/// Pages the dataplane fetches ahead for one direct read. Clients send no -/// credits, so this window alone bounds the plaintext a direct read pins. +/// The fewest pages the dataplane fetches ahead for one direct read. Clients +/// send no credits, so this window, or `range_window_pages` when larger, alone +/// bounds the pages a direct read pins. pub const DIRECT_WINDOW_PAGES: usize = 4; /// One page handed to a [`DirectSink`]. diff --git a/cmd/racer-dataplane/src/read.rs b/cmd/racer-dataplane/src/read.rs index 3913908e9..146301101 100644 --- a/cmd/racer-dataplane/src/read.rs +++ b/cmd/racer-dataplane/src/read.rs @@ -190,14 +190,16 @@ impl Coordinator { .await?; validate_metadata(&metadata, &origin.object, &selector)?; // Direct reads have no client credits. The dataplane paces them - // with a fixed window and releases each page after writing it. + // with its own window, at least `range_window_pages`, and releases + // each page after writing it. + let direct_window = crate::client::DIRECT_WINDOW_PAGES.max(self.streams.window_pages()); let kind = match kind { ReadKind::HostRead { range, .. } | ReadKind::GpuRead { range, .. } => { ReadKind::Subscription { pin: None, range, - page_credits: crate::client::DIRECT_WINDOW_PAGES, - byte_credits: crate::client::DIRECT_WINDOW_PAGES as u64 * PAGE_BYTES, + page_credits: direct_window, + byte_credits: direct_window as u64 * PAGE_BYTES, } } kind => kind, diff --git a/cmd/racer-dataplane/src/read/range_stream.rs b/cmd/racer-dataplane/src/read/range_stream.rs index 25a1670b9..59b358f8c 100644 --- a/cmd/racer-dataplane/src/read/range_stream.rs +++ b/cmd/racer-dataplane/src/read/range_stream.rs @@ -44,7 +44,8 @@ use std::task::Waker; use std::time::Duration; use std::time::Instant; -/// Direct fills a read keeps in flight beyond its head page. +/// Direct fills a read keeps in flight beyond its head page. A sealed read +/// fills its whole window directly (see `RangeStream::direct_ahead`). const DIRECT_AHEAD: usize = 1; /// Provider selections a read keeps in flight. Beyond the first, a read only @@ -614,6 +615,11 @@ impl RangeStreams { } } + /// The configured `range_window_pages` of each stream. + pub(crate) fn window_pages(&self) -> usize { + self.window_pages + } + /// Share the same metadata and page owners used by newly opened streams. pub(crate) fn directory(&self) -> &Arc { &self.directory @@ -985,7 +991,7 @@ impl RangeStream { .any(|slot| matches!(slot, Slot::Fetching(_) | Slot::FetchingSealed(_))) } - /// Start a direct fill for the head page, up to `DIRECT_AHEAD` more holes, + /// Start a direct fill for the head page, up to `direct_ahead` more holes, /// and provider selections for the remaining holes. fn admit(&mut self, scope: &RequestScope) -> Result { // A failed page stops admission at and after it until it reaches the @@ -1024,7 +1030,8 @@ impl RangeStream { .map(WindowLease::busy_elsewhere) .unwrap_or_default(); let mut index = 1; - while ahead < DIRECT_AHEAD && index < limit { + let direct_ahead = self.direct_ahead(); + while ahead < direct_ahead && index < limit { if matches!(self.slots[index], Slot::Missing { .. }) && !elsewhere.contains(&self.page_at(index)) { @@ -1044,6 +1051,18 @@ impl RangeStream { Ok(changed) } + /// Direct fills to keep in flight beyond the head page. Provider selections + /// deliver verified plaintext, so a sealed read fills its whole window + /// directly. Its direct fills stage disk pages with cuFile and keep + /// ciphertext for the GPU to decrypt. + fn direct_ahead(&self) -> usize { + if self.sealed.is_some() { + self.window_pages.saturating_sub(1).max(DIRECT_AHEAD) + } else { + DIRECT_AHEAD + } + } + /// Returns false when the range budget admits no more work for now. fn start_fetch(&mut self, index: usize, scope: &RequestScope) -> Result { let Some(child) = self.budget.next_page(self.pending())? else { @@ -1738,6 +1757,28 @@ pub(super) mod tests { } } + #[test] + fn sealed_reads_fill_their_whole_window_directly() { + for (window, sealed) in [(1, 1), (2, 1), (8, 7)] { + let f = Fixture::new(crate::test_support::cluster::config(false).limits, window); + let mut metadata = metadata(); + metadata.length = 16 * PAGE_BYTES; + let range = ByteRange::From(0).resolve(metadata.length).unwrap(); + let scope = + RequestScope::new(RequestId([9; 16]), Instant::now() + Duration::from_secs(60)) + .unwrap(); + let mut stream = f.open(&metadata, range, scope); + assert_eq!(stream.direct_ahead(), DIRECT_AHEAD); + stream + .set_sealed(SealedTarget { + helper: crate::gpu::testing::helper(&[None]), + ordinal: 0, + }) + .unwrap(); + assert_eq!(stream.direct_ahead(), sealed, "window {window}"); + } + } + #[test] fn sealed_pages_are_delivered_in_order_only_to_direct_readers() { let f = Fixture::new(crate::test_support::cluster::config(false).limits, 2); diff --git a/docs/content/reference/racer.md b/docs/content/reference/racer.md index 8992ff053..43b95c70c 100644 --- a/docs/content/reference/racer.md +++ b/docs/content/reference/racer.md @@ -1068,6 +1068,15 @@ other slots hold pages read with cuFile. When all of them are in use, the dataplane reads the next page through host memory instead. If the image has no GPU decrypt support, the CPU decrypts every page. +A GPU read fetches its whole `RACER_RANGE_WINDOW_PAGES` window directly, with +cuFile where it can, so a larger window keeps more disk reads in flight. To +load from fast local disks, raise the window, raise `RACER_GPU_STAGING_BYTES` +to hold every page in flight, and raise the client's reads in flight. Each IO +worker's share of `RACER_PLAINTEXT_BYTES` and `RACER_CIPHERTEXT_BYTES` must +still hold the window plus one page. On one NVMe drive, a window of 8, 1 GiB +of staging, and 8 loader reads loaded a 7B model in vLLM in 2.34 seconds, +against 2.44 seconds with the defaults. That is close to the drive's speed. + The dataplane does not request `nvidia.com/gpu`. It is privileged, so it already sees `/dev/nvidia*`. It only needs the host driver libraries. `RACER_DATAPLANE_NVIDIA` in `racer-config` controls how it asks for them: