diff --git a/README.md b/README.md index e05a8218cf..c281c98dea 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ The reply states which baseline it used, and profiles the other framework does n | Gateway | `gateway-64`, `gateway-h3` | Reverse proxy + server stack over HTTP/2 and HTTP/3 with mixed workload | | Production Stack | `production-stack` | Four-service architecture: edge + Redis + JWT auth sidecar + server, 10K-item cache-aside, concurrent reads + writes | | WebSocket | `echo-ws`, `echo-ws-pipeline`, `echo-ws-limited` | Echo throughput across connection counts; 16x batched echo; echo with each connection closed after 10 messages (upgrade-handshake cost) | +| Interaction | `rendezvous` | `GET /rendezvous/{key}?v={value}`: two connections meet on a key, neither gets an answer until both have arrived, and each is answered with the other's value. Measures what crossing between workers costs, on its own board | ## Run Locally @@ -105,6 +106,7 @@ Swap `actix` for your entry and `h1` for the family you want: | Gateway | `gw` | reverse-proxy and production-stack profiles | | gRPC | `grpc` | unary and streaming, plaintext and TLS | | WebSocket | `ws` | the echo profiles | +| Interaction | `ix` | the Rendezvous profile | `` is your entry's name on the board, lowercased, with anything outside `A-Z a-z 0-9 . _ -` turned into `-`, the same name as your file in diff --git a/frameworks/actix/Cargo.lock b/frameworks/actix/Cargo.lock index e6ba2043c8..75a770357b 100644 --- a/frameworks/actix/Cargo.lock +++ b/frameworks/actix/Cargo.lock @@ -769,6 +769,7 @@ dependencies = [ "rustls-pemfile", "serde", "serde_json", + "tokio", "tokio-postgres", ] diff --git a/frameworks/actix/Cargo.toml b/frameworks/actix/Cargo.toml index 29db3568b4..1605f8a956 100644 --- a/frameworks/actix/Cargo.toml +++ b/frameworks/actix/Cargo.toml @@ -17,6 +17,7 @@ num_cpus = "1" tokio-postgres = { version = "0.7", features = ["with-serde_json-1"] } deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] } futures-util = "0.3" +tokio = { version = "1", features = ["sync"] } [profile.release] opt-level = 3 diff --git a/frameworks/actix/README.md b/frameworks/actix/README.md index 6dfa35b121..630ba6170c 100644 --- a/frameworks/actix/README.md +++ b/frameworks/actix/README.md @@ -14,6 +14,7 @@ Actix-web 4 HTTP server with rustls for TLS/HTTP/2 support, compiled with thin L | Endpoint | Method | Description | |----------|--------|-------------| | `/pipeline` | GET | Returns `ok` (plain text) | +| `/rendezvous/{key}` | GET | Waits for a second request on the same key, answers each with the other's `v` | | `/baseline11` | GET | Sums query parameter values | | `/baseline11` | POST | Sums query parameters + request body | | `/baseline2` | GET | Sums query parameter values (HTTP/2 variant) | diff --git a/frameworks/actix/meta.json b/frameworks/actix/meta.json index 21a3493c75..e901551b7a 100644 --- a/frameworks/actix/meta.json +++ b/frameworks/actix/meta.json @@ -26,7 +26,8 @@ "static-tls", "async-db", "baseline-h2", - "static-h2" + "static-h2", + "rendezvous" ], "maintainers": [] } diff --git a/frameworks/actix/src/main.rs b/frameworks/actix/src/main.rs index fe51bc8915..4c271bef10 100644 --- a/frameworks/actix/src/main.rs +++ b/frameworks/actix/src/main.rs @@ -18,6 +18,7 @@ use actix_web::http::header; use actix_web::http::header::{ContentType, HeaderValue, SERVER}; use actix_web::middleware::Compress; use actix_web::{web, App, HttpRequest, HttpResponse, HttpServer}; +use dashmap::mapref::entry::Entry; use dashmap::DashMap; use deadpool_postgres::{Manager, ManagerConfig, Pool as PgPool, RecyclingMethod}; use futures_util::StreamExt; @@ -156,6 +157,46 @@ fn load_dataset() -> Vec { // actix-web re-exports its runtime, so the sleep comes from there rather than from a direct // tokio dependency the entry does not have. It parks the task rather than the thread, so the // waits in flight are bounded by memory rather than by the worker count. +#[derive(Deserialize)] +struct RendezvousQuery { + v: String, +} + +// Actix runs one worker per core and a handler stays on the worker that accepted its +// connection, so the two sides of a key usually sit on different workers. The first +// parks on a oneshot, and the second wakes it across workers. +static RENDEZVOUS: std::sync::LazyLock)>> = + std::sync::LazyLock::new(DashMap::new); + +async fn rendezvous(path: web::Path, query: web::Query) -> HttpResponse { + let key = path.into_inner(); + let mut value = query.into_inner().v; + let other = loop { + match RENDEZVOUS.entry(key.clone()) { + Entry::Occupied(waiting) => { + let (peer_value, reply) = waiting.remove(); + match reply.send(value) { + Ok(()) => break peer_value, + // That client left while it waited: nobody to meet, so wait instead. + Err(returned) => value = returned, + } + } + Entry::Vacant(free) => { + let (reply, wait) = tokio::sync::oneshot::channel(); + free.insert((value, reply)); + match wait.await { + Ok(peer_value) => break peer_value, + Err(_) => return HttpResponse::InternalServerError().finish(), + } + } + } + }; + HttpResponse::Ok() + .insert_header((SERVER, SERVER_HDR.clone())) + .content_type(ContentType::plaintext()) + .body(other) +} + async fn delay(path: web::Path) -> HttpResponse { let ms = path.into_inner(); if ms > 0 { @@ -591,6 +632,7 @@ async fn main() -> io::Result<()> { .app_data(crud_cache.clone()) .route("/pipeline", web::get().to(pipeline)) .route("/delay/{ms}", web::get().to(delay)) + .route("/rendezvous/{key}", web::get().to(rendezvous)) .route("/baseline11", web::get().to(baseline11_get)) .route("/baseline11", web::post().to(baseline11_post)) .route("/baseline2", web::get().to(baseline2)) diff --git a/frameworks/aspnet-minimal/Handlers.cs b/frameworks/aspnet-minimal/Handlers.cs index 36dc7ccd00..a682faa25c 100644 --- a/frameworks/aspnet-minimal/Handlers.cs +++ b/frameworks/aspnet-minimal/Handlers.cs @@ -2,6 +2,7 @@ using HttpArena.Services; using HttpArena.Types; using System.Buffers; +using System.Collections.Concurrent; static class Handlers { @@ -16,6 +17,35 @@ public static async ValueTask SumBody(int a, int b, HttpRequest req) public static string Text() => "ok"; + // GET /rendezvous/{key}?v= - the first request for a key waits on a completion + // source; the second takes it out of the map, completes it with its own value and + // answers with the one the first left behind. + sealed class RendezvousWaiter(string value) + { + public readonly string Value = value; + public readonly TaskCompletionSource Reply = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + static readonly ConcurrentDictionary RendezvousWaiting = new(); + + public static ValueTask Rendezvous(string key, string v) + { + RendezvousWaiter? mine = null; + while (true) + { + if (RendezvousWaiting.TryRemove(key, out var peer)) + { + peer.Reply.TrySetResult(v); + return new ValueTask(peer.Value); + } + mine ??= new RendezvousWaiter(v); + if (RendezvousWaiting.TryAdd(key, mine)) + { + return new ValueTask(mine.Reply.Task); + } + } + } + // GET /delay/{ms} - answer after ms milliseconds. Task.Delay registers a // timer and yields; the thread goes back to the pool instead of sitting on // the request, so the number of waits in flight is bounded by memory rather diff --git a/frameworks/aspnet-minimal/Program.cs b/frameworks/aspnet-minimal/Program.cs index bd81159031..ccb9f2a6b1 100644 --- a/frameworks/aspnet-minimal/Program.cs +++ b/frameworks/aspnet-minimal/Program.cs @@ -116,6 +116,7 @@ app.MapPost("/echo", Handlers.EchoBody); app.MapGet("/delay/{ms:int}", Handlers.Delay); +app.MapGet("/rendezvous/{key}", Handlers.Rendezvous); app.MapGet("/json/{count}", Handlers.Json); app.MapGet("/async-db", Handlers.AsyncDatabase); diff --git a/frameworks/aspnet-minimal/README.md b/frameworks/aspnet-minimal/README.md index 0a064b9195..1a03251e41 100644 --- a/frameworks/aspnet-minimal/README.md +++ b/frameworks/aspnet-minimal/README.md @@ -14,6 +14,7 @@ Minimal ASP.NET Core HTTP server using .NET 10 with Kestrel and minimal API rout | Endpoint | Method | Description | |---|---|---| | `/pipeline` | GET | Returns `ok` (plain text) | +| `/rendezvous/{key}` | GET | Waits for a second request on the same key, answers each with the other's `v` | | `/baseline11` | GET | Sums query parameter values | | `/baseline11` | POST | Sums query parameters + request body | | `/baseline2` | GET | Sums query parameter values (HTTP/2 variant) | diff --git a/frameworks/aspnet-minimal/meta.json b/frameworks/aspnet-minimal/meta.json index 6aa4094393..8404cee8c9 100644 --- a/frameworks/aspnet-minimal/meta.json +++ b/frameworks/aspnet-minimal/meta.json @@ -29,7 +29,8 @@ "baseline-h2c", "json-h2c", "baseline-h3", - "static-h3" + "static-h3", + "rendezvous" ], "maintainers": [] } diff --git a/frameworks/chi/README.md b/frameworks/chi/README.md index ac968aa4fc..49a1ccfb5b 100644 --- a/frameworks/chi/README.md +++ b/frameworks/chi/README.md @@ -13,6 +13,7 @@ chi router on the Go `net/http` server, default configuration. | Endpoint | Method | Description | |----------|--------|-------------| | `/pipeline` | GET | Returns `ok` (plain text) | +| `/rendezvous/{key}` | GET | Waits for a second request on the same key, answers each with the other's `v` | | `/baseline11` | GET | Sums query parameter values | | `/baseline11` | POST | Sums query parameters + request body | | `/json/{count}?m=N` | GET | First `count` dataset items with `total = price * quantity * m` | diff --git a/frameworks/chi/main.go b/frameworks/chi/main.go index a4621c0ab4..4a206ee41f 100644 --- a/frameworks/chi/main.go +++ b/frameworks/chi/main.go @@ -10,6 +10,7 @@ import ( "runtime" "strconv" "strings" + "sync" "time" "github.com/go-chi/chi/v5" @@ -80,6 +81,45 @@ func delay(w http.ResponseWriter, r *http.Request) { w.Write([]byte(strconv.Itoa(ms))) } +type rendezvousWaiter struct { + value string + reply chan string +} + +var rendezvousWaiters sync.Map + +// The first request for a key parks its goroutine on a channel; the second finds it, +// hands over its own value and answers with the one the first left behind. +func rendezvous(w http.ResponseWriter, r *http.Request) { + key := chi.URLParam(r, "key") + value := r.URL.Query().Get("v") + other := "" + var mine *rendezvousWaiter + for { + if found, ok := rendezvousWaiters.LoadAndDelete(key); ok { + peer := found.(*rendezvousWaiter) + peer.reply <- value + other = peer.value + break + } + if mine == nil { + // Buffered, so the side completing the pair never blocks on this one. + mine = &rendezvousWaiter{value: value, reply: make(chan string, 1)} + } + if _, loaded := rendezvousWaiters.LoadOrStore(key, mine); !loaded { + select { + case other = <-mine.reply: + case <-r.Context().Done(): + rendezvousWaiters.CompareAndDelete(key, mine) + return + } + break + } + } + w.Header().Set("Content-Type", "text/plain") + w.Write([]byte(other)) +} + func baseline11(w http.ResponseWriter, r *http.Request) { sum := 0 for _, values := range r.URL.Query() { @@ -463,6 +503,7 @@ func main() { r.Get("/pipeline", pipeline) r.Get("/delay/{ms}", delay) + r.Get("/rendezvous/{key}", rendezvous) r.Get("/baseline11", baseline11) r.Post("/baseline11", baseline11) r.Get("/json/{count}", jsonItems) diff --git a/frameworks/chi/meta.json b/frameworks/chi/meta.json index 8e3788293c..d09e8311b6 100644 --- a/frameworks/chi/meta.json +++ b/frameworks/chi/meta.json @@ -24,7 +24,8 @@ "json-tls", "8gbit", "static-tls", - "async-db" + "async-db", + "rendezvous" ], "maintainers": [] } diff --git a/frameworks/helidon-production/meta.json b/frameworks/helidon-production/meta.json index 6738ad3bc0..42cbf27f30 100644 --- a/frameworks/helidon-production/meta.json +++ b/frameworks/helidon-production/meta.json @@ -31,7 +31,8 @@ "echo-ws-limited", "unary-grpc", "unary-grpc-tls", - "async-db" + "async-db", + "rendezvous" ], "maintainers": [ "tomas-langer" diff --git a/frameworks/helidon-production/src/main/java/com/httparena/Main.java b/frameworks/helidon-production/src/main/java/com/httparena/Main.java index 38f22f209c..6d7861fec8 100644 --- a/frameworks/helidon-production/src/main/java/com/httparena/Main.java +++ b/frameworks/helidon-production/src/main/java/com/httparena/Main.java @@ -33,6 +33,7 @@ public static void main(String[] args) throws Exception { .get("/pipeline", new PipelineHandler()) .get("/baseline11", baselineHandler) .get("/delay/{ms}", new DelayHandler()) + .get("/rendezvous/{key}", new RendezvousHandler()) .get("/json/{count}", jsonHandler) .get("/json", jsonHandler) .post("/baseline11", new BaselinePostHandler()) diff --git a/frameworks/helidon-production/src/main/java/com/httparena/RendezvousHandler.java b/frameworks/helidon-production/src/main/java/com/httparena/RendezvousHandler.java new file mode 100644 index 0000000000..d1f010226d --- /dev/null +++ b/frameworks/helidon-production/src/main/java/com/httparena/RendezvousHandler.java @@ -0,0 +1,48 @@ +package com.httparena; + +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; + +import io.helidon.webserver.http.Handler; +import io.helidon.webserver.http.ServerRequest; +import io.helidon.webserver.http.ServerResponse; + +import static io.helidon.http.HeaderValues.CONTENT_TYPE_TEXT_PLAIN; + +/** + * GET /rendezvous/{key}?v= - the first request for a key waits, the second answers both. + * Each request has a virtual thread of its own, so waiting is just blocking on a future. + */ +final class RendezvousHandler implements Handler { + private record Waiter(String value, CompletableFuture reply) {} + + private final ConcurrentHashMap waiting = new ConcurrentHashMap<>(); + + @Override + public void handle(ServerRequest req, ServerResponse res) throws Exception { + String key = req.path().pathParameters().get("key"); + String value = req.query().first("v").orElse(""); + + Waiter mine = null; + String other; + while (true) { + Waiter peer = waiting.remove(key); + if (peer != null) { + peer.reply().complete(value); + other = peer.value(); + break; + } + if (mine == null) { + mine = new Waiter(value, new CompletableFuture<>()); + } + if (waiting.putIfAbsent(key, mine) == null) { + other = mine.reply().get(); + break; + } + } + + res.header(CONTENT_TYPE_TEXT_PLAIN); + res.send(other.getBytes(StandardCharsets.ISO_8859_1)); + } +} diff --git a/frameworks/node/README.md b/frameworks/node/README.md index b3d83174c3..84ce9f50f1 100644 --- a/frameworks/node/README.md +++ b/frameworks/node/README.md @@ -14,6 +14,7 @@ Node's own HTTP server, `node:http`, with no framework on top and no dependencie | Endpoint | Method | Description | |----------|--------|-------------| | `/pipeline` | GET | Returns `ok` (plain text) | +| `/rendezvous/{key}` | GET | Waits for a second request on the same key, answers each with the other's `v` | | `/baseline11` | GET/POST | Sums query parameter values, plus the body for POST | | `/baseline2` | GET | Sums query parameter values | | `/json/:count` | GET | Serializes a slice of the dataset, gzipped when the client accepts it | diff --git a/frameworks/node/meta.json b/frameworks/node/meta.json index bd36d466ce..d51b950f48 100644 --- a/frameworks/node/meta.json +++ b/frameworks/node/meta.json @@ -15,7 +15,8 @@ "8gbit", "static-tls", "async-db", - "fortunes" + "fortunes", + "rendezvous" ], "maintainers": [] } diff --git a/frameworks/node/server.js b/frameworks/node/server.js index de58b40cb6..33d9465cd4 100644 --- a/frameworks/node/server.js +++ b/frameworks/node/server.js @@ -29,7 +29,25 @@ function getCPUCount() { if (cluster.isPrimary) { const numCPUs = getCPUCount(); - for (let i = 0; i < numCPUs; i++) cluster.fork(); + + // rendezvous: the workers are separate processes and the two sides of a key usually + // land on different ones, so the pairing lives here and every request crosses IPC. + const waiting = new Map(); + const onRendezvous = (worker, m) => { + const peer = waiting.get(m.key); + if (!peer) { + waiting.set(m.key, { worker, id: m.id, value: m.value }); + return; + } + waiting.delete(m.key); + peer.worker.send({ id: peer.id, value: m.value }); + worker.send({ id: m.id, value: peer.value }); + }; + + for (let i = 0; i < numCPUs; i++) { + const worker = cluster.fork(); + worker.on('message', (m) => onRendezvous(worker, m)); + } } else { // A missing dataset serves an empty list instead of taking the worker down let datasetItems = []; @@ -337,6 +355,15 @@ if (cluster.isPrimary) { } } + // The primary answers a rendezvous by the id it was sent with. + const parked = new Map(); + let nextParked = 0; + process.on('message', (m) => { + const res = parked.get(m.id); + parked.delete(m.id); + if (res) sendText(res, m.value); + }); + const handle = (req, res) => { // req.url is the request target, so the path is everything before the "?" const url = req.url; @@ -346,6 +373,13 @@ if (cluster.isPrimary) { if (path === '/pipeline') return sendText(res, 'ok'); + if (path.startsWith('/rendezvous/')) { + const id = nextParked++; + parked.set(id, res); + process.send({ id, key: path.slice(12), value: queryValue(query, 'v') }); + return; + } + if (path === '/baseline11') { const querySum = sumQuery(query); if (req.method !== 'POST') return sendText(res, String(querySum)); diff --git a/frameworks/vertx/README.md b/frameworks/vertx/README.md index cbbaf105de..21239f9401 100644 --- a/frameworks/vertx/README.md +++ b/frameworks/vertx/README.md @@ -13,6 +13,7 @@ Eclipse Vert.x with vertx-web on Netty, one server verticle per core. | Endpoint | Method | Description | |----------|--------|-------------| | `/pipeline` | GET | Returns `ok` (plain text) | +| `/rendezvous/{key}` | GET | Waits for a second request on the same key, answers each with the other's `v` | | `/baseline11` | GET | Sums query parameter values | | `/baseline11` | POST | Sums query parameters + request body | | `/json/{count}?m=N` | GET | First `count` dataset items with `total = price * quantity * m` | diff --git a/frameworks/vertx/meta.json b/frameworks/vertx/meta.json index 2a9323eaf6..8cdb5e4e6a 100644 --- a/frameworks/vertx/meta.json +++ b/frameworks/vertx/meta.json @@ -24,7 +24,8 @@ "json-tls", "8gbit", "static-tls", - "async-db" + "async-db", + "rendezvous" ], "maintainers": [] } diff --git a/frameworks/vertx/src/main/java/httparena/ServerVerticle.java b/frameworks/vertx/src/main/java/httparena/ServerVerticle.java index e4f9efa786..7d817e5419 100644 --- a/frameworks/vertx/src/main/java/httparena/ServerVerticle.java +++ b/frameworks/vertx/src/main/java/httparena/ServerVerticle.java @@ -6,7 +6,9 @@ import java.util.List; import java.util.ArrayList; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import io.vertx.core.Context; import io.vertx.core.buffer.Buffer; import io.vertx.core.Future; import io.vertx.core.VerticleBase; @@ -92,6 +94,7 @@ public Future start() { vertx.setTimer(ms, id -> text(ctx, Long.toString(ms))); } }); + router.get("/rendezvous/:key").handler(this::rendezvous); router.get("/json/:count").handler(this::jsonItems); router.get("/async-db").handler(this::asyncDb); router.get("/static/:filename").handler(this::staticFile); @@ -132,6 +135,37 @@ public Future start() { return plain; } + private record RendezvousWaiter(String value, RoutingContext ctx, Context owner) {} + + // Shared by every verticle: there is one per core, and the two sides of a key + // usually sit on different event loops. + private static final ConcurrentHashMap RENDEZVOUS = new ConcurrentHashMap<>(); + + private void rendezvous(RoutingContext ctx) { + String key = ctx.pathParam("key"); + String value = ctx.queryParams().get("v"); + RendezvousWaiter mine = null; + while (true) { + RendezvousWaiter peer = RENDEZVOUS.remove(key); + if (peer != null) { + // The parked response belongs to its own event loop, so it is answered there. + peer.owner().runOnContext(v -> { + if (!peer.ctx().response().closed()) { + text(peer.ctx(), value); + } + }); + text(ctx, peer.value()); + return; + } + if (mine == null) { + mine = new RendezvousWaiter(value, ctx, vertx.getOrCreateContext()); + } + if (RENDEZVOUS.putIfAbsent(key, mine) == null) { + return; + } + } + } + private void text(RoutingContext ctx, String body) { ctx.response().putHeader("content-type", "text/plain").end(body); } diff --git a/requests/rendezvous.lua b/requests/rendezvous.lua new file mode 100644 index 0000000000..e90a12020c --- /dev/null +++ b/requests/rendezvous.lua @@ -0,0 +1,38 @@ +-- wrk script for the rendezvous profile: GET /rendezvous/{key}?v={value}. +-- +-- Each thread numbers its requests and gives every two in a row the same key. A +-- connection holds one request at a time, and the first of a pair cannot be answered +-- until the second arrives, so the two always go out on different connections. Which +-- two is decided by which connections happen to be free, so a server cannot arrange +-- for a pair to land on the same worker: the key only shows up once they are both in. +-- +-- Nothing can deadlock as long as a thread has two connections. Only the newest +-- request can be missing its partner, every older pair completes, and that frees a +-- connection to send the partner. +-- +-- Keys are never reused: the nonce separates runs against the same container, so a +-- waiter left behind by the previous run's exit cannot pair with this run's request. + +local threads = 0 +local nonce = os.time() + +function setup(thread) + threads = threads + 1 + thread:set("id", threads) + thread:set("nonce", nonce) +end + +function init(args) + prefix = "GET /rendezvous/" .. nonce .. "-" .. id .. "-" + suffix = " HTTP/1.1\r\nHost: " .. wrk.host .. ":" .. wrk.port .. "\r\n\r\n" + -- wrk calls request() once on the first thread to check the script, before any + -- connection opens, and that call never reaches the server. Start one early there + -- so it does not take the first half of a pair with it. + n = (id == 1) and -1 or 0 +end + +request = function() + local i = n + n = n + 1 + return prefix .. math.floor(i / 2) .. "?v=" .. id .. "x" .. (i + 1) .. suffix +end diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 642846464b..955754267e 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -169,7 +169,7 @@ for t in baseline pipelined limited-conn json-comp json-tls upload \ baseline-h2 static-h2 baseline-h2c json-h2c \ baseline-h3 static-h3 \ unary-grpc unary-grpc-tls \ - echo-ws echo-ws-pipeline echo-ws-limited; do + echo-ws echo-ws-pipeline echo-ws-limited rendezvous; do if framework_subscribes_to "$t"; then _has_isolated_test=true; break; fi done $_has_isolated_test && framework_build diff --git a/scripts/check_badge_parity.js b/scripts/check_badge_parity.js index 1084d7ee86..09501df55c 100644 --- a/scripts/check_badge_parity.js +++ b/scripts/check_badge_parity.js @@ -99,7 +99,7 @@ const run = new Function('D', ` `)(D); // ── rank the same way write_badges() does ─────────────────────────────────── -const FAMILIES = ['h1', 'h2', 'h3', 'gw', 'grpc', 'ws']; +const FAMILIES = ['h1', 'h2', 'h3', 'gw', 'grpc', 'ws', 'ix']; const LEAGUES = [['flagship', 'emerging'], ['engine'], ['experimental'], ['infrastructure']]; const MIN_FIELD = 2; diff --git a/scripts/gen_leaderboard_data.py b/scripts/gen_leaderboard_data.py index c7a11011b1..e4390e7276 100644 --- a/scripts/gen_leaderboard_data.py +++ b/scripts/gen_leaderboard_data.py @@ -129,6 +129,13 @@ ("gateway-h3", "Gateway (H3)", "Reverse proxy + server over h3.", [64,256], [64,256], True,True,False), ("production-stack", "Production Stack", "Edge + Redis + JWT auth + server.",[256,1024],[256,1024],True,True,False), ]), + ("Interaction", [ + # Every other profile answers a request from that request alone. Here one + # connection's request cannot be answered without another's, so the shared- + # nothing designs that lead the other boards have to pay for the crossing (#1686). + ("rendezvous", "Rendezvous", "Each request waits for another connection's, and gets its value.", + [4096],[4096],True,True,False), + ]), ("WebSocket", [ ("echo-ws", "Echo", "WebSocket echo throughput.", [512,4096,16384],[512,4096,16384],True,True,False), ("echo-ws-pipeline", "Echo Pipelined", "Batched WebSocket echo.", [512,4096,16384],[512,4096,16384],True,True,False), @@ -169,6 +176,7 @@ "latency-500k-8cpu": "test-profiles/h1/latency-500k-8cpu/implementation", "async-db": "test-profiles/h1/async-database/implementation", "fortunes": "test-profiles/h1/fortunes/implementation", + "rendezvous": "test-profiles/interaction/rendezvous/implementation", "baseline-h2": "test-profiles/h2/baseline-h2/implementation", "static-h2": "test-profiles/h2/static-h2/implementation", "baseline-h2c": "test-profiles/h2/baseline-h2c/implementation", @@ -1252,9 +1260,9 @@ def _slug(name): # category -> family key. Mirrors familyOf() in index.html; anything unlisted is # HTTP/1.1 (Connection, Workload, Database, Multi-endpoint). FAMILY_OF_CAT = {"Gateway": "gw", "HTTP/2": "h2", "HTTP/3": "h3", - "gRPC": "grpc", "WebSocket": "ws"} + "gRPC": "grpc", "WebSocket": "ws", "Interaction": "ix"} FAMILY_LABEL = {"h1": "H/1.1", "h2": "H/2", "h3": "H/3", - "gw": "Gateway", "grpc": "gRPC", "ws": "WebSocket"} + "gw": "Gateway", "grpc": "gRPC", "ws": "WebSocket", "ix": "Interaction"} # Leagues are the board's type filter. flagship+emerging is its default view and # ranks as one field; engine and infrastructure entries are each scored on their @@ -1846,16 +1854,18 @@ def compute_achievements(agg, profiles, meta, badge_index): OG_MUTED = (154, 160, 166) SCOPE_NAME = {"h1": "HTTP/1.1", "h2": "HTTP/2", "h3": "HTTP/3", - "gw": "Gateway", "grpc": "gRPC", "ws": "WebSocket"} + "gw": "Gateway", "grpc": "gRPC", "ws": "WebSocket", "ix": "Interaction"} # The board's

per family. renderHead() writes the same strings, so the # pre-rendered heading and the rendered one match. SCOPE_H1 = {"h1": "HTTP/1.1 web server benchmarks", "h2": "HTTP/2 web server benchmarks", "h3": "HTTP/3 web server benchmarks", "gw": "Gateway and production stack benchmarks", - "grpc": "gRPC server benchmarks", "ws": "WebSocket server benchmarks"} + "grpc": "gRPC server benchmarks", "ws": "WebSocket server benchmarks", + "ix": "Cross-connection web server benchmarks"} SCOPE_BLURB = {"h1": "all HTTP/1.1 profiles", "h2": "all HTTP/2 profiles", "h3": "all HTTP/3 profiles", "gw": "the gateway & production-stack profiles", - "grpc": "all gRPC profiles", "ws": "all WebSocket profiles"} + "grpc": "all gRPC profiles", "ws": "all WebSocket profiles", + "ix": "the interaction profiles"} # The board's own default: view=composite, scope=h1, flagship+emerging, tuned # shown, useMem and rescale off. Prerendering anything else would show a crawler # a page no visitor lands on. @@ -2760,7 +2770,8 @@ def _lang_faq(lang, scopes, n_entries, round_name): continue fw, _k, _t, score, rank, of, _l, _lr, _lo = rows[0] qa.append(( - f"What is the fastest {lang} {SCOPE_NAME[scope]} server?", + f"What is the fastest {lang} " + f"{'cross-connection' if scope == 'ix' else SCOPE_NAME[scope]} server?", f"{fw}, with a composite of {score:.0f} on {SCOPE_NAME[scope]} and " f"{rank} of {of} against all languages in its league.")) qa.append(( diff --git a/scripts/lib/profiles.sh b/scripts/lib/profiles.sh index 2707f2bcf0..ef2c0846d2 100644 --- a/scripts/lib/profiles.sh +++ b/scripts/lib/profiles.sh @@ -108,6 +108,13 @@ declare -A PROFILES=( [echo-ws]="1|0|0-31,64-95|512,4096,16384|ws-echo" [echo-ws-pipeline]="16|0|0-31,64-95|512,4096,16384|ws-echo" [echo-ws-limited]="1|10|0-31,64-95|512,4096|ws-echo" + # Rendezvous: GET /rendezvous/{key}?v={value} (requests/rendezvous.lua). A request + # cannot be answered until another connection's request for the same key arrives, + # and each gets the other's value. Every other profile answers a request from that + # request alone, which is the case a shared-nothing event loop per core is built + # for; here the two sides usually sit on different workers, so something has to + # cross between them on every exchange. + [rendezvous]="1|0|0-31,64-95|4096|rendezvous" ) PROFILE_ORDER=( @@ -123,6 +130,7 @@ PROFILE_ORDER=( production-stack unary-grpc unary-grpc-tls echo-ws echo-ws-pipeline echo-ws-limited + rendezvous latency-1m latency-10k latency-500k-8cpu # Last on purpose. It closes ~49K sockets at exit and every one sits in # TIME_WAIT for the kernel's fixed ~60s, so anything scheduled after it @@ -149,7 +157,7 @@ parse_profile() { endpoint_tool() { case "$1" in # wrk (lua script rotation) - static-tls|json-tls) echo "wrk" ;; + static-tls|json-tls|rendezvous) echo "wrk" ;; # zrk — the only paced generator; holds a fixed offered rate latency-1m|latency-10k|latency-500k-8cpu|8gbit) echo "zrk" ;; # h2load for all HTTP/2 variants (TLS via ALPN + h2c prior-knowledge) diff --git a/scripts/lib/tools/wrk.sh b/scripts/lib/tools/wrk.sh index 6f42eeeae7..ad61e8c127 100644 --- a/scripts/lib/tools/wrk.sh +++ b/scripts/lib/tools/wrk.sh @@ -33,6 +33,13 @@ wrk_build_args() { -s "$REQUESTS_DIR/json-tls-rotate.lua" "https://localhost:$H1TLS_PORT") ;; + rendezvous) + # Plain HTTP/1.1. The script hands out the keys that pair requests up, + # which is why this is wrk: the pairing has to be chosen per request. + cmd+=(-t "$THREADS" -c "$conns" -d "$duration" + -s "$REQUESTS_DIR/rendezvous.lua" + "http://localhost:$PORT") + ;; 8gbit) # POST /echo, 100 KB in and the same 100 KB back. wrk reports only # the bytes it READ, so the Transfer/sec it prints is the download diff --git a/scripts/validate-rendezvous.py b/scripts/validate-rendezvous.py new file mode 100644 index 0000000000..c3d330efbf --- /dev/null +++ b/scripts/validate-rendezvous.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Rendezvous validation for HttpArena: GET /rendezvous/{key}?v={value}. + +The first request for a key waits, the second completes both, and each is answered +with the other's value. The benchmark's generator reads none of that back, so this is +the whole anti-cheat: a server that answers without waiting, or that pairs requests by +anything but the key, has to fail here. + +Zero-dependency HTTP/1.1 client on raw asyncio streams. Keys and values are drawn fresh +on every run, so nothing can be prepared for them. + +Usage: python3 validate-rendezvous.py [host] [port] + Defaults: localhost 8080 + Exit code 0 = all passed, 1 = failures +""" + +import asyncio +import random +import secrets +import string +import sys +import time + +HOST = sys.argv[1] if len(sys.argv) > 1 else "localhost" +PORT = int(sys.argv[2]) if len(sys.argv) > 2 else 8080 + +# A lone request has to still be waiting after this long. The profile's own waits are +# microseconds, so anything answered before this did not wait for its partner. +HOLD_SECONDS = 0.4 +# How long a completed pair may take. Generous: this is a correctness check on a +# possibly small hosted runner, not a performance one. +REPLY_TIMEOUT = 15.0 +# Enough connections at once that they spread across every worker a server runs, so a +# per-worker map loses most of its pairs rather than passing by luck. +MANY_PAIRS = 256 + +PASS = 0 +FAIL = 0 + + +def result(label, ok, detail=""): + global PASS, FAIL + if ok: + PASS += 1 + print(f" PASS [{label}]{' (' + detail + ')' if detail else ''}") + else: + FAIL += 1 + print(f" FAIL [{label}]{': ' + detail if detail else ''}") + + +def token(n=16): + return "".join(secrets.choice(string.ascii_letters + string.digits) for _ in range(n)) + + +class Conn: + """One keep-alive connection, so a test can issue several requests on it.""" + + async def open(self): + self.reader, self.writer = await asyncio.open_connection(HOST, PORT) + return self + + def send(self, key, value): + self.writer.write( + f"GET /rendezvous/{key}?v={value} HTTP/1.1\r\n" + f"Host: {HOST}:{PORT}\r\n\r\n".encode() + ) + + async def recv(self): + """Read one response. Returns (status, content-type, body).""" + head = await self.reader.readuntil(b"\r\n\r\n") + lines = head.decode("latin-1").split("\r\n") + status = int(lines[0].split(" ")[1]) + headers = {} + for line in lines[1:]: + if ":" in line: + name, val = line.split(":", 1) + headers[name.strip().lower()] = val.strip() + if headers.get("transfer-encoding", "").lower() == "chunked": + body = b"" + while True: + size = int((await self.reader.readline()).strip().split(b";")[0], 16) + if size == 0: + await self.reader.readline() + break + body += await self.reader.readexactly(size) + await self.reader.readexactly(2) + else: + body = await self.reader.readexactly(int(headers.get("content-length", "0"))) + return status, headers.get("content-type", ""), body.decode("latin-1") + + async def reply(self): + return await asyncio.wait_for(self.recv(), REPLY_TIMEOUT) + + def close(self): + self.writer.close() + + +async def exchange_values(): + """Each side of a pair gets the other's value, verbatim, as text/plain.""" + a, b = await Conn().open(), await Conn().open() + key = token() + # Lengths at both ends of what the contract allows, so nothing fixed-size passes. + va, vb = token(1), token(32) + a.send(key, va) + await asyncio.sleep(0.05) + b.send(key, vb) + try: + ra, rb = await asyncio.gather(a.reply(), b.reply()) + except (asyncio.TimeoutError, asyncio.IncompleteReadError, ConnectionError, ValueError) as e: + result("exchange", False, f"no complete answer: {type(e).__name__}") + a.close(), b.close() + return + result("first request gets the second's value", ra[0] == 200 and ra[2] == vb, + f"status {ra[0]}, expected {vb!r}, got {ra[2]!r}" if ra[2] != vb or ra[0] != 200 else "") + result("second request gets the first's value", rb[0] == 200 and rb[2] == va, + f"status {rb[0]}, expected {va!r}, got {rb[2]!r}" if rb[2] != va or rb[0] != 200 else "") + result("Content-Type", "text/plain" in ra[1] and "text/plain" in rb[1], f"{ra[1]!r}, {rb[1]!r}") + a.close(), b.close() + + +async def first_request_waits(): + """A request with no partner is not answered until one arrives.""" + a, b = await Conn().open(), await Conn().open() + key, va, vb = token(), token(), token() + a.send(key, va) + pending = asyncio.ensure_future(a.recv()) + await asyncio.sleep(HOLD_SECONDS) + held = not pending.done() + result("a lone request waits", held, + "" if held else f"answered within {HOLD_SECONDS}s with no second request") + if not held: + pending.cancel() + a.close(), b.close() + return + started = time.monotonic() + b.send(key, vb) + try: + ra, rb = await asyncio.wait_for(asyncio.gather(pending, b.recv()), REPLY_TIMEOUT) + ok = ra[2] == vb and rb[2] == va + result("the second request releases it", ok, + f"{time.monotonic() - started:.3f}s" if ok else f"got {ra[2]!r} and {rb[2]!r}") + except (asyncio.TimeoutError, asyncio.IncompleteReadError, ConnectionError, ValueError) as e: + result("the second request releases it", False, f"no answer: {type(e).__name__}") + a.close(), b.close() + + +async def many_pairs_at_once(): + """Hundreds of keys in flight together: every side gets its own partner's value.""" + conns = await asyncio.gather(*(Conn().open() for _ in range(MANY_PAIRS * 2))) + sends = [] + for i in range(MANY_PAIRS): + key = token() + sends.append((conns[2 * i], key, token(), conns[2 * i + 1])) + # One side of every pair first, in random order, then all the others, so every pair + # is split in time as well as across connections. + random.shuffle(sends) + expected = {} + for first, key, value, partner in sends: + first.send(key, value) + expected[first] = (key, value, partner) + await asyncio.sleep(0.1) + values = {} + random.shuffle(sends) + for first, key, value, partner in sends: + pv = token() + values[partner] = pv + partner.send(key, pv) + + want = {} + for first, (key, value, partner) in expected.items(): + want[first] = values[partner] + want[partner] = value + + async def one(conn): + try: + return (await conn.reply())[2] == want[conn] + except (asyncio.TimeoutError, asyncio.IncompleteReadError, ConnectionError, ValueError): + return None + + outcomes = await asyncio.gather(*(one(c) for c in conns)) + wrong = sum(1 for o in outcomes if o is False) + missing = sum(1 for o in outcomes if o is None) + result(f"{MANY_PAIRS} pairs at once", wrong == 0 and missing == 0, + f"{wrong} got someone else's value, {missing} never answered" + if wrong or missing else "every side got its partner's value") + for c in conns: + c.close() + + +async def keep_alive_and_key_reuse(): + """Connections stay usable, and a key is free again once its pair has completed.""" + a, b = await Conn().open(), await Conn().open() + key = token() + ok = True + detail = "" + for round_no in range(20): + va, vb = token(), token() + a.send(key, va) + await asyncio.sleep(0) + b.send(key, vb) + try: + ra, rb = await asyncio.gather(a.reply(), b.reply()) + except (asyncio.TimeoutError, asyncio.IncompleteReadError, ConnectionError, ValueError) as e: + ok, detail = False, f"round {round_no}: no answer: {type(e).__name__}" + break + if ra[2] != vb or rb[2] != va: + ok, detail = False, f"round {round_no}: stale values {ra[2]!r}, {rb[2]!r}" + break + result("same key, same connections, 20 rounds", ok, detail) + a.close(), b.close() + + +async def disconnected_waiter(): + """A client that leaves while it waits must not take the server down with it.""" + gone = await Conn().open() + gone.send(token(), token()) + await asyncio.sleep(0.05) + gone.close() + await asyncio.sleep(0.1) + + a, b = await Conn().open(), await Conn().open() + key, va, vb = token(), token(), token() + a.send(key, va) + b.send(key, vb) + try: + ra, rb = await asyncio.gather(a.reply(), b.reply()) + ok = ra[2] == vb and rb[2] == va + result("still serving after a waiter disconnects", ok, "" if ok else f"got {ra[2]!r}, {rb[2]!r}") + except (asyncio.TimeoutError, asyncio.IncompleteReadError, ConnectionError, ValueError) as e: + result("still serving after a waiter disconnects", False, type(e).__name__) + a.close(), b.close() + + +async def main(): + for test in (exchange_values, first_request_waits, many_pairs_at_once, + keep_alive_and_key_reuse, disconnected_waiter): + try: + await test() + except OSError as e: + result(test.__name__, False, f"could not connect: {e}") + + +if __name__ == "__main__": + asyncio.run(main()) + print(f" {PASS} passed, {FAIL} failed") + sys.exit(1 if FAIL else 0) diff --git a/scripts/validate.sh b/scripts/validate.sh index f10af4413c..5061fa568e 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -2469,6 +2469,23 @@ if has_test "echo-ws" || has_test "echo-ws-pipeline" || has_test "echo-ws-limite fi fi +if has_test "rendezvous"; then + RENDEZVOUS_DOCS="$DOCS_BASE/interaction/rendezvous/validation" + echo "[test] rendezvous endpoint" + RV_OUTPUT=$(python3 "$SCRIPT_DIR/validate-rendezvous.py" localhost "$PORT" 2>&1) || true + echo "$RV_OUTPUT" + + RV_PASS=$(grep -oP '(\d+) passed' <<< "$RV_OUTPUT" | grep -oP '\d+' || true) + RV_FAIL=$(grep -oP '(\d+) failed' <<< "$RV_OUTPUT" | grep -oP '\d+' || true) + # No summary line means the script itself died, which must not read as a pass. + if [ -z "$RV_PASS$RV_FAIL" ]; then RV_FAIL=1; fi + PASS=$((PASS + ${RV_PASS:-0})) + FAIL=$((FAIL + ${RV_FAIL:-0})) + if [ "${RV_FAIL:-0}" -gt 0 ]; then + echo " → $RENDEZVOUS_DOCS" + fi +fi + # ───── Gateway profiles (reverse proxy + server, shared validation flow) ───── # # Both gateway-64 (h2) and gateway-h3 (h3 at the edge) use the same endpoint diff --git a/site/content/docs/add-framework/badges.md b/site/content/docs/add-framework/badges.md index 094477b46f..ffe4c6abd8 100644 --- a/site/content/docs/add-framework/badges.md +++ b/site/content/docs/add-framework/badges.md @@ -32,6 +32,7 @@ under `scopes..default.markdown`. | Gateway | `gw` | reverse-proxy and production-stack profiles | | gRPC | `grpc` | unary and streaming, plaintext and TLS | | WebSocket | `ws` | the echo profiles | +| Interaction | `ix` | the Rendezvous profile | `` is your entry's board name, lowercased, with anything outside `A-Z a-z 0-9 . _ -` replaced by `-`. It is the same name as your file under diff --git a/site/content/docs/running-locally/scripts/benchmark.md b/site/content/docs/running-locally/scripts/benchmark.md index f0e96ced63..f78c99d070 100644 --- a/site/content/docs/running-locally/scripts/benchmark.md +++ b/site/content/docs/running-locally/scripts/benchmark.md @@ -140,6 +140,7 @@ pipeline | req_per_conn | cpu_limit | connections | endpoint | `unary-grpc-tls` | 1 | ∞ | `0-31,64-95` | 256, 1024 | h2load | same, TLS | | `gateway-64` | 1 | ∞ | `0-31,64-95` | 256, 1024 | h2load | 20-URI mix behind nginx via `docker compose` | | `echo-ws` | 1 | ∞ | `0-31,64-95` | 512, 4096, 16384 | gcannon `--ws` | `/ws` | +| `rendezvous` | 1 | ∞ | `0-31,64-95` | 4096 | wrk | `/rendezvous/{key}?v=` via `rendezvous.lua` | **Subscription:** a framework only runs profiles listed in its `meta.json` `tests` array. Profiles it's not subscribed to are silently skipped. diff --git a/site/content/docs/scoring/composite-score.md b/site/content/docs/scoring/composite-score.md index 4c147e9ea8..f66aead58a 100644 --- a/site/content/docs/scoring/composite-score.md +++ b/site/content/docs/scoring/composite-score.md @@ -115,6 +115,14 @@ Not all profiles count toward the composite score. Profiles marked as **scored** | Echo Pipelined | Yes | Batched WebSocket echo throughput | | Echo Short-lived | Yes | WebSocket echo with each connection closed after 10 messages | +### Interaction + +| Profile | Scored | Workload | +|---|---|---| +| Rendezvous | Yes | Two connections meet on a key, and each is answered with the other's value | + +Rendezvous ranks on its own board and counts toward no other composite. + Fortunes, Pipelined, Static TLS and Async DB are the reference-only profiles - shown on the board as faded columns for comparison, but not counted in the composite score. The two database profiles were scored until recently. They stopped because the database and its driver dominate them far more than the framework does: a framework's `async-db` number mostly reports which Postgres driver its language has, which is not what this board sets out to compare. They are still run and still published, because the number is worth having; it just no longer decides the ranking. @@ -171,7 +179,7 @@ B actually wins the memory term despite A's 5× throughput advantage, because `s Types are scored **separately** - each has its own composite ranking and normalization pool. The scored profiles differ by type: -- **Frameworks** (Flagship, Emerging and Experimental, in either Standard or Tuned mode) are scored on all scored profiles across H/1.1, H/2, H/3, gRPC, and WebSocket. +- **Frameworks** (Flagship, Emerging and Experimental, in either Standard or Tuned mode) are scored on all scored profiles across H/1.1, H/2, H/3, gRPC, WebSocket and Interaction. - **Engines** are scored on a reduced set: Baseline, Short-lived, JSON TLS, async-db, H/2 (all three), H/3 (both), gRPC (all four), the gateway profiles, and WebSocket, since most engines don't implement the heavier endpoints (compression negotiation, the TLS echo). - **Infrastructure** (nginx, Caddy, h2o and similar proxies / static-file servers) is scored on the nine profiles a server can answer without an application framework behind it: Baseline, Pipelined, Short-lived, JSON TLS, Static TLS, and Baseline and Static over both H/2 and H/3. Upload, the database profiles, gRPC, WebSocket and the gateway stacks may be displayed as reference data but do not count. diff --git a/site/content/docs/test-profiles/_index.md b/site/content/docs/test-profiles/_index.md index 374cc9c423..52ff5b34a0 100644 --- a/site/content/docs/test-profiles/_index.md +++ b/site/content/docs/test-profiles/_index.md @@ -29,7 +29,7 @@ Five load generators are dispatched per profile - each one is built for a specif | Parameter | Value | |-----------|-------| -| Load generators | `gcannon` (HTTP/1.1, WebSocket), `wrk` (static, json-tls and 8gbit rotation), `h2load` (HTTP/2, h2c, gateway), `h2load-h3` (HTTP/3 / QUIC), `zrk` (fixed-rate) | +| Load generators | `gcannon` (HTTP/1.1, WebSocket), `wrk` (static, json-tls, 8gbit and rendezvous rotation), `h2load` (HTTP/2, h2c, gateway), `h2load-h3` (HTTP/3 / QUIC), `zrk` (fixed-rate) | | Threads | 64 for `gcannon` / `wrk` / `h2load` / `h2load-h3` (`$THREADS` / `$H2THREADS` / `$H3THREADS`) | | Duration | 5s default; `async-db` 10s (hardcoded in the profile dispatcher) | | Runs | 3 per (profile, connection count) - best RPS wins | @@ -63,4 +63,5 @@ Gateway and `production-stack` profiles are compose-orchestrated, so their servi {{< card link="h3" title="H/3" subtitle="Baseline and static file benchmarks over QUIC for frameworks with native H/3 support." icon="globe-alt" >}} {{< card link="grpc" title="gRPC" subtitle="Unary RPC throughput over cleartext HTTP/2 using Protocol Buffers serialization." icon="globe-alt" >}} {{< card link="ws" title="WebSocket" subtitle="WebSocket echo throughput measuring frame processing performance." icon="globe-alt" >}} + {{< card link="interaction" title="Interaction" subtitle="Connections that depend on each other: no request can be answered from that request alone, so work has to cross between workers." icon="switch-horizontal" >}} {{< /cards >}} diff --git a/site/content/docs/test-profiles/interaction/_index.md b/site/content/docs/test-profiles/interaction/_index.md new file mode 100644 index 0000000000..ab210670a9 --- /dev/null +++ b/site/content/docs/test-profiles/interaction/_index.md @@ -0,0 +1,14 @@ +--- +weight: 6 +title: Interaction +seo_title: "Interaction Test Profiles" +description: "Benchmarks where connections depend on each other: no request can be answered from that request alone, so work has to cross between workers." +--- + +Every other profile answers a request from that request alone. That suits a design with one event loop per core and nothing shared, where connections never need each other, and it is why those designs lead most boards. + +Plenty of real services don't work like that. A chat server, a game server or a matching engine has to get a message from one connection to another, and the two usually sit on different workers. Interaction profiles measure what that crossing costs, so they rank on a board of their own and count toward no other composite. + +{{< cards >}} + {{< card link="rendezvous" title="Rendezvous" subtitle="Two connections meet on a key. Neither gets an answer until both have arrived, and each is answered with the other's value." icon="switch-horizontal" >}} +{{< /cards >}} diff --git a/site/content/docs/test-profiles/interaction/rendezvous/_index.md b/site/content/docs/test-profiles/interaction/rendezvous/_index.md new file mode 100644 index 0000000000..aa7527b78f --- /dev/null +++ b/site/content/docs/test-profiles/interaction/rendezvous/_index.md @@ -0,0 +1,12 @@ +--- +title: Rendezvous +seo_title: "Rendezvous Benchmark: Cross-Connection Interaction" +description: "Two connections meet on a key: each request waits for the other and is answered with its value, measuring what crossing between workers costs a server." +--- + +Two connections meet on a key. Neither gets an answer until both have arrived, and each is answered with the other's value. + +{{< cards >}} + {{< card link="implementation" title="Implementation Guidelines" subtitle="Endpoint specification, expected behaviour, and type-specific rules." icon="code" >}} + {{< card link="validation" title="Validation" subtitle="All checks executed by the validation script for this test profile." icon="check-circle" >}} +{{< /cards >}} diff --git a/site/content/docs/test-profiles/interaction/rendezvous/implementation.md b/site/content/docs/test-profiles/interaction/rendezvous/implementation.md new file mode 100644 index 0000000000..21bb1fb37a --- /dev/null +++ b/site/content/docs/test-profiles/interaction/rendezvous/implementation.md @@ -0,0 +1,60 @@ +--- +title: Implementation Guidelines +seo_title: "Rendezvous Benchmark: Implementation Guide" +description: "Endpoint contract, pairing rules, and the constraints a framework must satisfy for the rendezvous benchmark." +--- +{{< type-rules standard="Must park the request with the framework's own async primitives and answer through its normal response path. The pairing lives in the server's own processes, with no broker or database." tuned="May use sharded maps, lock-free structures, custom wakeups or direct socket writes, as long as the contract holds. The pairing still lives in the server's own processes." engine="The pairing lives in the server's own processes, with no broker or database." >}} + + +The Rendezvous profile measures what it costs a server to get something from one connection to another. A request cannot be answered until another connection's request for the same key arrives, so no response can be produced from one request alone. + +This is scored on the Interaction board only, and counts toward no other composite. + +**Connections:** 4,096 + +## How it works + +``` +GET /rendezvous/{key}?v={value} HTTP/1.1 +``` + +1. The first request for a key waits, and is not answered yet. +2. The second request for that key completes both: each is answered `200` with the other's `value` as the body. +3. The key is free again, and a later pair may use it. + +| Part | Contract | +|---|---| +| `key` | Opaque, `[A-Za-z0-9-]`, up to 64 characters | +| `v` | Opaque, `[A-Za-z0-9]`, 1 to 32 characters. Returned verbatim, so don't parse it as a number | +| Response | `200`, `Content-Type: text/plain`, body is the other request's `v` | + +A key never has more than two requests on it at once. A request waits until its partner arrives or its client leaves, with no deadline of its own. + +## What it measures + +- **The crossing.** With one worker per core, the two sides usually sit on different workers, so every exchange has to hand something between them: a queue, a wakeup, a lock, a copy or a message to another process. +- **Shared state under contention.** All 4,096 connections go through whatever holds the waiting requests. +- **Parked requests.** Up to half of the connections can be waiting on a partner at once, so what a waiting request costs counts too. + +## Why a server can't arrange its own pairs + +The generator is `wrk` with `requests/rendezvous.lua`. Each thread gives every two consecutive requests the same key, and a connection only holds one request at a time, so the two go out on whichever connections are free. A server can't move a pair onto the same worker, because it only learns which two connections are paired once they have both sent. Keys are never reused between runs. + +## Multi-process servers + +The two sides of a key usually land on different processes when a server forks one per core, so the pairing has to work across them. IPC or shared memory are fine. An external service is not: Redis or a database in the path would measure that service instead. + +## Blocking servers + +A server with a fixed pool of blocking workers deadlocks here if the pool is smaller than the number of waiting requests: every worker holds a request waiting for a partner, and no worker is free to read the partner. Such a server can leave this profile out of `meta.json.tests`. + +## Parameters + +| Parameter | Value | +|-----------|-------| +| Endpoint | `GET /rendezvous/{key}?v={value}` | +| Connections | 4,096 | +| Pipeline | 1 | +| Duration | 5s | +| Runs | 3 (best taken) | +| Load generator | wrk + `requests/rendezvous.lua` | diff --git a/site/content/docs/test-profiles/interaction/rendezvous/validation.md b/site/content/docs/test-profiles/interaction/rendezvous/validation.md new file mode 100644 index 0000000000..facc8a8825 --- /dev/null +++ b/site/content/docs/test-profiles/interaction/rendezvous/validation.md @@ -0,0 +1,17 @@ +--- +title: Validation +seo_title: "Rendezvous Benchmark: Validation Checks" +description: "Every check the validation script runs against the rendezvous endpoint, and the failure each one catches." +--- + +The load generator doesn't read response bodies, so these checks are the whole anti-cheat. They run from `scripts/validate-rendezvous.py`, with keys and values drawn fresh on every run. + +| Check | What it does | What it catches | +|---|---|---| +| Each side gets the other's value | Two requests on one key with values of 1 and 32 characters. Each must be answered `200` with the other's value, verbatim | Echoing your own value, a fixed-size buffer, parsing `v` as a number | +| Content-Type | Both responses must be `text/plain` | The wrong response path | +| A lone request waits | One request, no partner: it must still be unanswered after 0.4s | Answering without waiting for the partner | +| The second request releases it | The partner arrives, and both must be answered | A waiter that never wakes | +| 256 pairs at once | 512 connections, one side of each pair first in random order, then the other sides. Every connection must get its own partner's value | Pairing by anything but the key, and a per-worker map, which loses any pair split across workers | +| Same key, 20 rounds | Two connections exchange 20 times on one key | A key that isn't freed, stale values, connections that don't stay usable | +| Still serving after a waiter disconnects | A client leaves while waiting, then a fresh pair must complete | A crash or a stuck worker when a parked response has nowhere to go | diff --git a/site/leaderboard/index.html b/site/leaderboard/index.html index 945155a24d..d52db21f73 100644 --- a/site/leaderboard/index.html +++ b/site/leaderboard/index.html @@ -744,13 +744,13 @@

Object.keys(D.results).forEach(function(k){ D.results[k].forEach(function(r){ if(r.lang) FWLANG[r.fw]=r.lang; }); }); var COMPOSITE = '__composite__'; - var SCOPES = [{k:'h1',label:'H/1.1'},{k:'h2',label:'H/2'},{k:'h3',label:'H/3'},{k:'gw',label:'Gateway'},{k:'grpc',label:'gRPC'},{k:'ws',label:'WebSocket'}]; - function familyOf(p){ return {'Gateway':'gw','HTTP/2':'h2','HTTP/3':'h3','gRPC':'grpc','WebSocket':'ws'}[p.category]||'h1'; } + var SCOPES = [{k:'h1',label:'H/1.1'},{k:'h2',label:'H/2'},{k:'h3',label:'H/3'},{k:'gw',label:'Gateway'},{k:'grpc',label:'gRPC'},{k:'ws',label:'WebSocket'},{k:'ix',label:'Interaction'}]; + function familyOf(p){ return {'Gateway':'gw','HTTP/2':'h2','HTTP/3':'h3','gRPC':'grpc','WebSocket':'ws','Interaction':'ix'}[p.category]||'h1'; } // cats must list every CATALOG category that belongs to the family. This is // the nav's only source of profiles, so a category missing here disappears // from Detailed Views entirely while still ranking on the composite -- // which is how async and latency-1m went missing when they were added. - var FAMILIES=[{k:'h1',label:'HTTP/1.1',cats:['Connection','Concurrency','Efficiency','Workload','Database']},{k:'h2',label:'HTTP/2',cats:['HTTP/2']},{k:'h3',label:'HTTP/3',cats:['HTTP/3']},{k:'gw',label:'Gateway',cats:['Gateway']},{k:'grpc',label:'gRPC',cats:['gRPC']},{k:'ws',label:'WebSocket',cats:['WebSocket']}]; + var FAMILIES=[{k:'h1',label:'HTTP/1.1',cats:['Connection','Concurrency','Efficiency','Workload','Database']},{k:'h2',label:'HTTP/2',cats:['HTTP/2']},{k:'h3',label:'HTTP/3',cats:['HTTP/3']},{k:'gw',label:'Gateway',cats:['Gateway']},{k:'grpc',label:'gRPC',cats:['gRPC']},{k:'ws',label:'WebSocket',cats:['WebSocket']},{k:'ix',label:'Interaction',cats:['Interaction']}]; var expanded=new Set(['composite']); // state.lang is an exact language filter, distinct from the q search box: q // matches substrings across name/language/engine, which is fine for typing but @@ -1389,10 +1389,10 @@

var cat=document.getElementById('pcat'), title=document.getElementById('ptitle'), blurb=document.getElementById('pblurb'), ctl=document.getElementById('headctl'); if(state.view==='composite'){ - var sLabel={h1:'all HTTP/1.1 profiles',h2:'all HTTP/2 profiles',h3:'all HTTP/3 profiles',gw:'the gateway & production-stack profiles',grpc:'all gRPC profiles',ws:'all WebSocket profiles'}[state.scope]; + var sLabel={h1:'all HTTP/1.1 profiles',h2:'all HTTP/2 profiles',h3:'all HTTP/3 profiles',gw:'the gateway & production-stack profiles',grpc:'all gRPC profiles',ws:'all WebSocket profiles',ix:'the interaction profiles'}[state.scope]; // The heading is the page's

. "HTTP/1.1" alone said nothing about // what is on the page, so every view names its subject. - var sName={h1:'HTTP/1.1 web server benchmarks',h2:'HTTP/2 web server benchmarks',h3:'HTTP/3 web server benchmarks',gw:'Gateway and production stack benchmarks',grpc:'gRPC server benchmarks',ws:'WebSocket server benchmarks'}[state.scope]; + var sName={h1:'HTTP/1.1 web server benchmarks',h2:'HTTP/2 web server benchmarks',h3:'HTTP/3 web server benchmarks',gw:'Gateway and production stack benchmarks',grpc:'gRPC server benchmarks',ws:'WebSocket server benchmarks',ix:'Cross-connection web server benchmarks'}[state.scope]; cat.textContent='Composite ranking'; title.textContent=sName; blurb.innerHTML='Normalized score summed across '+sLabel+' (each profile worth 100 to its leader' +(state.rescale?' among the frameworks shown':' in the full field, so filtering does not change the numbers')