From d1dc362d21ec08ad8ada8c8f7d3a4d79e4856f3b Mon Sep 17 00:00:00 2001 From: nick Date: Thu, 5 Nov 2026 16:49:28 +0800 Subject: [PATCH] feat(benchmarks): report port resources at connection failure --- benchmarks/CONNECTIONS.md | 9 +- benchmarks/CONNECTIONS_CN.md | 9 +- benchmarks/asio_connections.cpp | 4 + benchmarks/connections.cpp | 6 + benchmarks/connections_ports.hpp | 173 +++++++++++++++++++++++++++ benchmarks/connections_resources.hpp | 118 ++++++++++++++++++ tests/connection_load_resources.cpp | 111 +++++++++++++++++ 7 files changed, 424 insertions(+), 6 deletions(-) create mode 100644 benchmarks/connections_ports.hpp diff --git a/benchmarks/CONNECTIONS.md b/benchmarks/CONNECTIONS.md index 36eaeb8..6f5576b 100644 --- a/benchmarks/CONNECTIONS.md +++ b/benchmarks/CONNECTIONS.md @@ -98,13 +98,13 @@ The buffer preflight reservation is `N * max-buffer-bytes + max-inflight-bytes + Large local client runs also consume ephemeral source ports. Closed TCP connections can remain in `TIME_WAIT`, so consecutive runs can exhaust the available ports and fail with `EADDRNOTAVAIL` even below the FD and memory limits. Record the OS port range and TCP states; allow previous connections to retire before an independent run, or distribute clients across suitable hosts/addresses. Preserve the failed run rather than counting it as a completed connection target. A local client's port limit is not the server's connection capacity. -Before every batch, check free source-port headroom against the connection target, including ports held in `TIME_WAIT`. Wait for natural release or fail preflight when headroom is insufficient; a previous process's clean exit does not imply that its source ports are reusable. +Before every batch, record the source-port range and available TCP-state observations. The configured range width is not free capacity: reserved ports, local addresses, destination tuples, other applications and reuse rules also matter. A namespace/host-wide `TIME_WAIT` count is not this process's count and must not be subtracted from the range to invent free headroom. If usable headroom cannot be established, record it as unavailable and wait for an independent run or fail preflight. A previous process's clean exit does not imply that its source ports are reusable. A ramp that fails with `EADDRNOTAVAIL` before reaching its target is separate from a TCP stall after all target connections have been established. There is one main scheduling thread and one cleanup-watchdog thread in addition to the configured IO workers. It does not create a thread for each connection. Ctrl+C or SIGTERM requests a failed run with normal cleanup. If the run, final output or cleanup cannot finish by the overall deadline plus cleanup grace, the watchdog requests immediate OS process termination: `TerminateProcess` with code 124 on Windows, or `SIGKILL` on POSIX. POSIX shells commonly display 137 for signal 9, but it is a signal termination, not a portable exit-code guarantee. This path means `clean_shutdown=false`; no complete success summary is produced. Termination does not depend on another log record, a C-runtime flush or sanitizer exit handlers. ## Output and interpretation -Stdout is JSON Lines: a `resource_startup` event for each process, a server `ready` event, periodic `sample` events and one final `result` event. The native Asio control also emits `resource_startup`. Startup identifiers retain each tool's existing convention: decimal strings in the arknet tool and integers in the native control. This uses the existing initial resource sample, before connection admission; it does not add a sampling pass or change the cadence. Consumers should select events by `event` and ignore additional fields. Stderr contains argument/startup errors. Watchdog termination may leave output incomplete, so also record each process's exit code. Save each process's streams separately outside the Git repository. +Stdout is JSON Lines: a `resource_startup` event for each process, a server `ready` event, periodic `sample` events, an optional `connect_resource_failure` before client cleanup, and one final `result` event. The native Asio control also emits `resource_startup`. Startup identifiers retain each tool's existing convention: decimal strings in the arknet tool and integers in the native control. This reuses the initial process-resource sample before connection admission; separate read-only port queries run at startup and the first connect failure without changing the periodic cadence. Consumers should select events by `event` and ignore additional fields. Stderr contains argument/startup errors. Watchdog termination may leave output incomplete, so also record each process's exit code. Save each process's streams separately outside the Git repository. | Result fields | Interpretation | | --- | --- | @@ -122,6 +122,9 @@ Stdout is JSON Lines: a `resource_startup` event for each process, a server `rea | `resource_sample_elapsed_s`, `resource_cpu_interval_s` | CPU-query midpoint relative to the start of this run, and the actual interval used for its CPU percentage. The interval is null when CPU percentage is unavailable. Neither is a cross-host timestamp. | | `resource_phase` | `running` for startup and periodic observations during ramp/hold/protocol drain; `after_cleanup` for the final resource reading, after endpoints and IO workers have stopped. Final current memory/counts are not load-time values. | | `resource_unavailable` | Object mapping each missing resource field to its diagnostic reason. A real zero stays numeric; reasons are diagnostic text, not stable error codes. | +| `network_resources` (startup) | `ephemeral_ports` contains inclusive `first`/`last` values, `range_source` and `range_scope`. macOS reads `net.inet.ip.portrange.first`/`last` through `sysctlbyname` (host scope); Linux reads `/proc/sys/net/ipv4/ip_local_port_range` (network-namespace scope). Unsupported or failed queries return null with an `unavailable` reason including numeric OS errors when a system query fails. No external command or setting change is performed. | +| `tcp_time_wait`, `time_wait_source`, `time_wait_scope` inside network resources | Linux reads only `/proc/net/sockstat`'s `TCP: tw` counter, scoped to all processes in the current network namespace. Other platforms explicitly report unavailable. These counters are neither per-process nor a count of occupied ephemeral ports for the selected destination; queries are not atomic with connection counters. | +| `connect_failure_resources` (`connect_resource_failure` event and result) | The main thread emits the failure event before client cleanup and repeats its captured snapshot in the final result. First connect failure captured before cleanup, with its numeric `error_code`, `error_category`, observed `established` count and `network_resources`. The `phase` remains `first_connect_failure_before_cleanup` even though the result is printed after cleanup. Later failures do not replace it. Null means no connect failure was observed; it does not explain a later traffic stall. Diagnostic capture failure is explicit. | | `sample_interval_ms` | Requested sampling cadence, included in the final result. Actual CPU intervals are reported separately. | | `content_errors`, `sequence_errors`, `send_errors`, `timeouts`, `resource_rejections` | Validation, transport, deadline or configured-budget failures. `shutdown_cancelled_sends` is separate from normal send errors. | | `clean_shutdown`, `outcome`, `first_error` | Cleanup status and `target_complete`, `observation_complete` or `failed`. Clean shutdown alone does not imply delivery success or a target was reached. | @@ -153,7 +156,7 @@ The JSON changes are additive; existing fields retain their types and measuremen Configure with `-DARKNET_BUILD_TESTS=ON` to also build `arknet_connection_protocol_test`. It checks identity preservation, corrupted/truncated frame rejection, overflow and mergeable RTT bucket bounds. `arknet_check` includes those checks and `--help`; the POSIX full-output-pipe process check verifies watchdog termination without relying on output consumption. Deployment load runs remain local/manual. -`arknet_connection_resources_test` checks the actual resource output fields, numeric zero versus missing metrics and reasons, diagnostic escaping, and independent periodic/whole-run time windows. The standalone [process resource probe](process_resources.md) covers the platform counters themselves. +`arknet_connection_resources_test` checks the actual resource output fields, numeric zero versus missing metrics and reasons, diagnostic escaping, independent periodic/whole-run time windows, validated port-range/TIME_WAIT parsing, native query errors and retention of the first pre-cleanup connect-failure snapshot. It uses injected failure counters, not port exhaustion. The standalone [process resource probe](process_resources.md) covers the process counters themselves. Before increasing scale, run both idle and traffic with a few connections, IPv4 and IPv6, and inspect both final results. Also verify a refused connection, mismatched run ID, target deadline, duplicate identity and an invalid buffer/window budget fail with a nonzero exit. Keep the command lines, executable/source identity, machine configuration, all failures and both process logs with each measurement. This tool's test protocol does not replace an ark application, authentication, reconnect recovery or cross-machine production acceptance. diff --git a/benchmarks/CONNECTIONS_CN.md b/benchmarks/CONNECTIONS_CN.md index 72145a4..ce07488 100644 --- a/benchmarks/CONNECTIONS_CN.md +++ b/benchmarks/CONNECTIONS_CN.md @@ -98,13 +98,13 @@ Windows 下由 CMake 提供系统网络与进程内存库。构建或使用 arkn 本地大量建连还会占用客户端临时端口。TCP 连接关闭后可能仍处于 `TIME_WAIT`；连续运行会耗尽可用端口，即使 FD 和内存尚未达到限额，也可能报 `EADDRNOTAVAIL`。记录系统端口范围和 TCP 状态，等上一轮连接释放后再独立运行，或将客户端分布到合适的主机与地址。失败记录应保留，不能算作完成了目标连接数；本地客户端的端口限制也不代表服务端容量。 -每轮开始前，都应将空闲源端口余量与连接目标比较，并计入 `TIME_WAIT` 占用。余量不足时等待自然释放，或让预检失败；上一轮进程清理成功，不代表源端口已经可以复用。 +每轮开始前，都应记录源端口范围和可用的 TCP 状态观察。配置范围的宽度不等于空闲容量：保留端口、本地地址、目标四元组、其他应用及复用规则都会影响可用性。命名空间／主机范围的 `TIME_WAIT` 数量不是当前进程的数量，不能从端口范围中直接扣减来推算空闲余量。无法确认可用余量时应记录为 unavailable，等待独立运行时机或让预检失败。上一轮进程清理成功，不代表源端口已经可以复用。尚未达到目标连接数就出现 `EADDRNOTAVAIL` 的建连失败，与全部目标连接建立后的 TCP 停滞必须分别记录。 除指定的 IO 工作线程外，每进程还有一条主调度线程和一条清理监控线程，不会为每条连接创建线程。Ctrl+C 或 SIGTERM 会请求失败退出，并执行正常清理。如果总期限加清理宽限期后，运行、最终输出或清理仍未完成，监控线程请求操作系统立即终止进程：Windows 使用 `TerminateProcess`，退出码为 124；POSIX 使用 `SIGKILL`。POSIX shell 通常将信号 9 显示为 137，但这不是跨平台统一退出码的保证。该路径表示 `clean_shutdown=false`，不产生完整成功摘要；终止过程不依赖再写日志、C 运行时的 flush 或 sanitizer 退出处理。 ## 输出与指标 -Stdout 使用 JSON Lines：每个进程的 `resource_startup`、服务端的 `ready`、定期 `sample` 和最终 `result`。原生 Asio 对照程序也输出 `resource_startup`。启动标识沿用各工具已有的格式：arknet 程序使用十进制字符串，原生对照程序使用整数。启动记录复用建连准入前已有的首次资源采样，不新增采样，也不改变采样周期。消费方应按 `event` 选择事件，并忽略新增字段。Stderr 用于参数或启动错误。清理超时中止可能留下不完整输出，因此还需记录进程退出码。分别保存各进程输出，原始结果放在 Git 仓库外。 +Stdout 使用 JSON Lines：每个进程的 `resource_startup`、服务端的 `ready`、定期 `sample`、客户端清理前可选的 `connect_resource_failure` 和最终 `result`。原生 Asio 对照程序也输出 `resource_startup`。启动标识沿用各工具已有的格式：arknet 程序使用十进制字符串，原生对照程序使用整数。启动记录复用建连准入前已有的首次进程资源采样；另在启动和首次建连失败时执行只读端口查询，不改变周期采样频率。消费方应按 `event` 选择事件，并忽略新增字段。Stderr 用于参数或启动错误。清理超时中止可能留下不完整输出，因此还需记录进程退出码。分别保存各进程输出，原始结果放在 Git 仓库外。 | 字段 | 口径 | | --- | --- | @@ -122,6 +122,9 @@ Stdout 使用 JSON Lines：每个进程的 `resource_startup`、服务端的 `re | `resource_sample_elapsed_s`、`resource_cpu_interval_s` | CPU 查询中点相对本次运行开始的秒数，以及 CPU 占用率实际使用的区间长度。CPU 占用率不可用时，区间为 null；这些值不能作为跨机器时间戳。 | | `resource_phase` | 启动和周期记录为 `running`，周期记录可能处于建连、保持或协议排空阶段；最终记录为 `after_cleanup`，在端点和 IO 工作线程停止后读取。最终内存和数量指标不是负载阶段的值。 | | `resource_unavailable` | 以缺失资源字段名为键、诊断原因为值的对象。真实零值仍是数字；原因文本用于诊断，不是稳定错误码。 | +| 启动记录的 `network_resources` | `ephemeral_ports` 含闭区间的 `first`／`last`，并附 `range_source` 和 `range_scope`。macOS 通过 `sysctlbyname` 读取 `net.inet.ip.portrange.first`／`last`，范围为整台主机；Linux 读取 `/proc/sys/net/ipv4/ip_local_port_range`，范围为当前网络命名空间。不支持或查询失败时为 null，`unavailable` 记录原因，系统查询失败时包含数字 OS 错误。不调用外部命令，不修改设置。 | +| 网络资源内的 `tcp_time_wait`、`time_wait_source`、`time_wait_scope` | Linux 仅读取 `/proc/net/sockstat` 的 `TCP: tw`，涵盖当前网络命名空间的所有进程；其他平台明确报告 unavailable。这既不是本进程计数，也不是特定目标已占用的临时端口数；系统查询与连接计数不是原子快照。 | +| `connect_resource_failure` 事件和最终记录的 `connect_failure_resources` | 主线程在客户端清理前输出失败事件，并在最终记录中重复保留的快照。保留清理前首次建连失败的数字 `error_code`、`error_category`、当时观察到的 `established` 和 `network_resources`。虽然最终结果在清理后输出，`phase` 仍为 `first_connect_failure_before_cleanup`。后续错误不覆盖它。Null 表示未观察到建连失败，不解释后续流量停滞；诊断采样自身失败也会明确记录。 | | `sample_interval_ms` | 最终结果中的计划采样间隔。实际 CPU 采样区间单独报告。 | | `content_errors`、`sequence_errors`、`send_errors`、`timeouts`、`resource_rejections` | 内容、序号、发送、期限或预算失败；停止过程中取消的发送单独记为 `shutdown_cancelled_sends`。 | | `clean_shutdown`、`outcome`、`first_error` | 清理状态，以及 `target_complete`、`observation_complete` 或 `failed`。清理完成本身不代表交付成功或达到连接目标。 | @@ -151,7 +154,7 @@ JSON 只新增字段，原有字段的类型和测量阶段保持不变。对比 配置 `-DARKNET_BUILD_TESTS=ON` 后可以构建 `arknet_connection_protocol_test`，检查身份保真、损坏与截断消息拒绝、长度溢出及可合并延迟桶边界。`arknet_check` 包含这些检查及 `--help`；POSIX 的满输出管道进程用例会检查监控线程在无人读取输出时仍可终止。部署负载仍在本地手动运行。 -`arknet_connection_resources_test` 检查实际资源输出字段、真实零值与缺失指标及其原因、诊断文本转义，以及周期和全程时间范围的独立性。系统计数器本身由独立的[进程资源检查程序](process_resources_CN.md)验证。 +`arknet_connection_resources_test` 检查实际资源输出字段、真实零值与缺失指标及其原因、诊断文本转义、周期和全程时间范围的独立性，以及端口范围／TIME_WAIT 解析、原生查询错误和首次清理前建连失败快照的保留。使用注入错误计数，不耗尽端口。进程计数器本身由独立的[进程资源检查程序](process_resources_CN.md)验证。 扩大规模之前，先用少量连接检查 idle、traffic、IPv4 和 IPv6 的双端最终计数，再验证拒绝连接、错误运行标识、目标期限、重复身份及非法缓冲或窗口预算均非零退出。每次测量保留命令、可执行文件与源码身份、机器配置、全部失败和双端日志。本工具的测试协议不能替代 ark 应用、鉴权、重连恢复或跨机器生产验收。 diff --git a/benchmarks/asio_connections.cpp b/benchmarks/asio_connections.cpp index 8cedaf6..dd95a94 100644 --- a/benchmarks/asio_connections.cpp +++ b/benchmarks/asio_connections.cpp @@ -572,6 +572,7 @@ struct application stopped_at{}, drained_at{}; clock_type::time_point sampled = began; connection_load::resource_observer resources{began}; + connection_load::connect_failure_observer connect_resources; }; session::session(application& owner, asio::io_context& context) @@ -1219,6 +1220,7 @@ void application::launch() { if (!stopping && error) { ++connect_errors; + connect_resources.capture(error.value(), error.category().name(), established.load()); fail("connect callback", error.message()); } else if (!stopping) @@ -1417,6 +1419,7 @@ void application::run() { { if (failed) { + connect_resources.write_event(std::cout, o.role, o.run, o.client); stop(); } else if (clock_type::now() >= until_deadline) @@ -1487,6 +1490,7 @@ void application::result() const { std::cout << ",\"tcp_kernel_samples_enabled\":true"; } } + connect_resources.write_fields(std::cout); std::cout << "}\n"; if (!failure.empty() || !clean || !complete) { diff --git a/benchmarks/connections.cpp b/benchmarks/connections.cpp index bf1d243..12c436a 100644 --- a/benchmarks/connections.cpp +++ b/benchmarks/connections.cpp @@ -311,6 +311,7 @@ struct counters outstanding{}, shutdown_cancelled_sends{}; std::atomic work_begin{}, work_end{}; connection_load::latency_distribution latency; + connection_load::connect_failure_observer connect_resources; std::mutex error_mutex; std::string first_error; @@ -531,6 +532,8 @@ class client_run } if (!values_.closing) { + values_.connect_resources.capture( + error.value(), error.category().name(), values_.established.load()); values_.fail("connect: " + error.message()); } return; @@ -572,6 +575,7 @@ class client_run { --values_.connecting; ++values_.connect_failures; + values_.connect_resources.capture(0, "connect_submission_rejected", values_.established.load()); values_.fail("connect submission rejected"); } } @@ -987,6 +991,7 @@ void result(const options& settings, const counters& values, std::string_view ou << ",\"buffer_budget_bytes\":" << settings.buffer_budget << ",\"sample_interval_ms\":" << settings.sample_ms << ",\"peak_pending_send_bytes\":" << values.peak_pending_bytes; connection_load::write_resource_fields(std::cout, resources, true); + values.connect_resources.write_fields(std::cout); std::cout << ",\"first_error\":" << quoted(values.first_error) << ",\"latency_observations\":" << values.latency.total(); for (const auto percent : {50, 95, 99}) @@ -1143,6 +1148,7 @@ int run(const options& settings) { } } active_before_stop = values.active; + values.connect_resources.write_event(std::cout, settings.role, settings.run, settings.client, true); client.stop(); } runtime.stop(); diff --git a/benchmarks/connections_ports.hpp b/benchmarks/connections_ports.hpp new file mode 100644 index 0000000..3d1462c --- /dev/null +++ b/benchmarks/connections_ports.hpp @@ -0,0 +1,173 @@ +// Copyright (c) 2026 OpenArkStudio. Distributed under BSL-1.0. See LICENSE. +#pragma once + +#include +#include +#include +#include + +#if defined(__APPLE__) +#include + +#endif + +#include "process_resources.hpp" + +namespace connection_load { + +struct ephemeral_port_range +{ + unsigned first{}, last{}; +}; + +struct port_resources +{ + arknet::benchmark::resource_metric ephemeral_ports; + std::string range_source = "unavailable", range_scope = "unavailable"; + arknet::benchmark::resource_metric tcp_time_wait; + std::string time_wait_source = "unavailable", time_wait_scope = "unavailable"; +}; + +namespace port_resource_detail { + +using arknet::benchmark::process_resource_detail::system_failure; +using arknet::benchmark::process_resource_detail::unavailable; + +inline auto checked_range(std::int64_t first, std::int64_t last) { + if (first < 1 || last > 65535 || first > last) + { + return unavailable("Invalid ephemeral port range returned by OS query"); + } + return arknet::benchmark::resource_metric{ + ephemeral_port_range{static_cast(first), static_cast(last)}, {}}; +} + +inline auto parse_port_range(const std::string& text) { + std::istringstream input(text); + std::int64_t first{}, last{}; + std::string extra; + if (!(input >> first >> last) || (input >> extra)) + { + return unavailable("Malformed ip_local_port_range query result"); + } + return checked_range(first, last); +} + +inline auto parse_time_wait(const std::string& text) { + std::istringstream input(text); + std::string line; + std::optional count; + bool found_tcp = false; + while (std::getline(input, line)) + { + std::istringstream fields(line); + std::string protocol, key, value; + fields >> protocol; + if (protocol != "TCP:") + { + continue; + } + if (found_tcp) + { + return unavailable("Duplicate TCP entry in /proc/net/sockstat"); + } + found_tcp = true; + while (fields >> key) + { + if (!(fields >> value)) + { + return unavailable("Truncated TCP entry in /proc/net/sockstat"); + } + if (key == "tw") + { + std::uint64_t parsed{}; + const auto [end, error] = std::from_chars(value.data(), value.data() + value.size(), parsed); + if (count || error != std::errc{} || end != value.data() + value.size()) + { + return unavailable("Invalid TCP tw entry in /proc/net/sockstat"); + } + count = parsed; + } + } + } + if (!count) + { + return unavailable("TCP tw entry unavailable in /proc/net/sockstat"); + } + return arknet::benchmark::resource_metric{count, {}}; +} + +inline arknet::benchmark::resource_metric read_query_file(const char* path) { + const auto file = std::unique_ptr(std::fopen(path, "rb"), &std::fclose); + if (!file) + { + const auto error = errno; + return unavailable(system_failure(("fopen(" + std::string(path) + ")").c_str(), error)); + } + char buffer[4096]; + const auto size = std::fread(buffer, 1, sizeof(buffer), file.get()); + if (std::ferror(file.get())) + { + const auto error = errno; + return unavailable(system_failure(("read(" + std::string(path) + ")").c_str(), error)); + } + if (size == sizeof(buffer)) + { + return unavailable("OS query exceeded the 4096-byte diagnostic bound"); + } + return {std::string(buffer, size), {}}; +} + +template +auto read_macos_range(Query query) { + int first{}, last{}; + for (const auto& [name, value] : + {std::pair{"net.inet.ip.portrange.first", &first}, std::pair{"net.inet.ip.portrange.last", &last}}) + { + std::size_t length = sizeof(*value); + if (query(name, value, &length) != 0) + { + const auto error = errno; + return unavailable( + system_failure(("sysctlbyname(" + std::string(name) + ")").c_str(), error)); + } + if (length != sizeof(*value)) + { + return unavailable( + "Unexpected integer size from sysctlbyname(" + std::string(name) + ")"); + } + } + return checked_range(first, last); +} + +} // namespace port_resource_detail + +inline port_resources read_port_resources() { + using namespace port_resource_detail; + port_resources result; + result.ephemeral_ports = + unavailable("Native ephemeral TCP port-range query is not implemented on this platform"); + result.tcp_time_wait = + unavailable("An inexpensive native TCP TIME_WAIT count is unavailable on this platform"); +#if defined(__APPLE__) + result.range_source = "sysctl:net.inet.ip.portrange.first,last"; + result.range_scope = "host"; + result.ephemeral_ports = read_macos_range([](const char* name, int* value, std::size_t* length) { + return sysctlbyname(name, value, length, nullptr, 0); + }); +#elif defined(__linux__) + result.range_source = "/proc/sys/net/ipv4/ip_local_port_range"; + result.range_scope = "network_namespace"; + const auto range = read_query_file(result.range_source.c_str()); + result.ephemeral_ports = + range.value ? parse_port_range(*range.value) : unavailable(range.unavailable_reason); + result.time_wait_source = "/proc/net/sockstat:TCP.tw"; + result.time_wait_scope = "network_namespace_all_processes"; + const auto sockets = read_query_file("/proc/net/sockstat"); + result.tcp_time_wait = + sockets.value ? parse_time_wait(*sockets.value) : unavailable(sockets.unavailable_reason); +#endif + return result; +} + +} // namespace connection_load diff --git a/benchmarks/connections_resources.hpp b/benchmarks/connections_resources.hpp index 9a7c53b..9489ef2 100644 --- a/benchmarks/connections_resources.hpp +++ b/benchmarks/connections_resources.hpp @@ -2,10 +2,12 @@ #pragma once #include +#include #include #include #include +#include "connections_ports.hpp" #include "process_resources.hpp" namespace connection_load { @@ -70,6 +72,120 @@ inline void write_resource_string(std::ostream& output, std::string_view value) output << '"'; } +inline void write_port_resources(std::ostream& output, const port_resources& sample) { + output << "{\"ephemeral_ports\":"; + if (sample.ephemeral_ports.value) + { + output << "{\"first\":" << sample.ephemeral_ports.value->first + << ",\"last\":" << sample.ephemeral_ports.value->last << '}'; + } + else + { + output << "null"; + } + output << ",\"range_source\":"; + write_resource_string(output, sample.range_source); + output << ",\"range_scope\":"; + write_resource_string(output, sample.range_scope); + output << ",\"tcp_time_wait\":"; + if (sample.tcp_time_wait.value) + { + output << *sample.tcp_time_wait.value; + } + else + { + output << "null"; + } + output << ",\"time_wait_source\":"; + write_resource_string(output, sample.time_wait_source); + output << ",\"time_wait_scope\":"; + write_resource_string(output, sample.time_wait_scope); + output << ",\"unavailable\":{\"ephemeral_ports\":"; + write_resource_string(output, sample.ephemeral_ports.unavailable_reason); + output << ",\"tcp_time_wait\":"; + write_resource_string(output, sample.tcp_time_wait.unavailable_reason); + output << "}}"; +} + +// First failure is sampled at the connect callback, before cleanup changes the +// socket population. Final output does not turn this into an after-cleanup count. +class connect_failure_observer +{ +public: + template + void capture(int error, std::string_view category, std::uint64_t established, Query query) noexcept { + try + { + const std::lock_guard lock(mutex_); + if (attempted_) + { + return; + } + attempted_ = true; + error_ = error; + established_ = established; + category_ = category; + sample_ = query(); + } + catch (...) + {} + } + + void capture(int error, std::string_view category, std::uint64_t established) noexcept { + capture(error, category, established, read_port_resources); + } + + void write_fields(std::ostream& output) const { + const std::lock_guard lock(mutex_); + write_fields_unlocked(output); + } + + void write_event(std::ostream& output, std::string_view role, std::uint64_t run, std::uint64_t client, + bool string_ids = false) const { + const std::lock_guard lock(mutex_); + if (!attempted_) + { + return; + } + output << "{\"event\":\"connect_resource_failure\",\"role\":"; + write_resource_string(output, role); + const auto quote = string_ids ? "\"" : ""; + output << ",\"run_id\":" << quote << run << quote << ",\"client_id\":" << quote << client << quote; + write_fields_unlocked(output); + output << "}\n" << std::flush; + } + +private: + void write_fields_unlocked(std::ostream& output) const { + output << ",\"connect_failure_resources\":"; + if (!attempted_) + { + output << "null"; + return; + } + output << "{\"phase\":\"first_connect_failure_before_cleanup\",\"established\":" << established_ + << ",\"error_code\":" << error_ << ",\"error_category\":"; + write_resource_string(output, category_); + output << ",\"network_resources\":"; + if (sample_) + { + write_port_resources(output, *sample_); + } + else + { + output << "{\"unavailable\":\"Failure resource snapshot could not be captured\"}"; + } + output << '}'; + } + + mutable std::mutex mutex_; + bool attempted_{}; + int error_{}; + std::uint64_t established_{}; + std::string category_; + std::optional sample_; +}; + inline void write_resource_fields(std::ostream& output, const resource_snapshot& sample, bool after_cleanup) { const auto metric = [&](std::string_view name, const auto& value) { output << ','; @@ -137,6 +253,8 @@ inline void write_resource_startup(std::ostream& output, const resource_snapshot const auto quote = string_ids ? "\"" : ""; output << ",\"run_id\":" << quote << run << quote << ",\"client_id\":" << quote << client << quote; write_resource_fields(output, sample, false); + output << ",\"network_resources\":"; + write_port_resources(output, read_port_resources()); output << "}\n" << std::flush; } diff --git a/tests/connection_load_resources.cpp b/tests/connection_load_resources.cpp index c0cbbcf..8912bc9 100644 --- a/tests/connection_load_resources.cpp +++ b/tests/connection_load_resources.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -43,6 +44,116 @@ void check_sample(const connection_load::resource_snapshot& snapshot) { } // namespace +DOCTEST_TEST_CASE("Port diagnostics validate OS ranges and namespace-wide TIME_WAIT without estimating headroom") { + using namespace connection_load::port_resource_detail; + const auto range = parse_port_range("49152 65535\n"); + DOCTEST_REQUIRE(range.value); + DOCTEST_CHECK(range.value->first == 49152); + DOCTEST_CHECK(range.value->last == 65535); + for (const auto text : {"", "0 65535", "49152 65536", "60000 50000", "-1 20", "1", "1 2 3", "1 x"}) + { + const auto invalid = parse_port_range(text); + DOCTEST_CHECK_FALSE(invalid.value); + DOCTEST_CHECK_FALSE(invalid.unavailable_reason.empty()); + } + for (const auto text : {"TCP: inuse 5 orphan 0 tw 0 alloc 7 mem 1\n", "UDP: inuse 4\nTCP: tw 0\n"}) + { + const auto zero = parse_time_wait(text); + DOCTEST_REQUIRE(zero.value); + DOCTEST_CHECK(*zero.value == 0); + } + DOCTEST_CHECK(parse_time_wait("TCP: inuse 1 tw 15000 mem 3\n").value == 15000); + for (const auto text : {"", "TCP6: tw 10", "TCP: inuse 5", "TCP: tw", "TCP: tw -1", "TCP: tw 1x", "TCP: tw 1 tw 2", + "TCP: tw 1\nTCP: tw 2", "TCP: tw 18446744073709551616"}) + { + DOCTEST_CHECK_FALSE(parse_time_wait(text).value); + } + const auto missing = read_query_file(""); + DOCTEST_CHECK_FALSE(missing.value); + DOCTEST_CHECK(missing.unavailable_reason.find("fopen() failed (OS error ") == 0); +} + +DOCTEST_TEST_CASE("Native port queries retain their source scope and explicit OS errors") { + using connection_load::port_resource_detail::read_macos_range; + const auto range = read_macos_range([](const char* name, int* value, std::size_t*) { + *value = std::string_view(name).ends_with("first") ? 49152 : 65535; + return 0; + }); + DOCTEST_REQUIRE(range.value); + DOCTEST_CHECK(range.value->first == 49152); + const auto failed = read_macos_range([](const char*, int*, std::size_t*) { + errno = EACCES; + return -1; + }); + DOCTEST_CHECK_FALSE(failed.value); + DOCTEST_CHECK(failed.unavailable_reason == + "sysctlbyname(net.inet.ip.portrange.first) failed (OS error " + std::to_string(EACCES) + ")"); + const auto short_result = read_macos_range([](const char*, int*, std::size_t* length) { + *length = 0; + return 0; + }); + DOCTEST_CHECK_FALSE(short_result.value); + const auto actual = connection_load::read_port_resources(); + if (actual.ephemeral_ports.value) + { + DOCTEST_CHECK(actual.ephemeral_ports.value->first > 0); + DOCTEST_CHECK(actual.ephemeral_ports.value->first <= actual.ephemeral_ports.value->last); + DOCTEST_CHECK(actual.ephemeral_ports.value->last <= 65535); + } + else + { + DOCTEST_CHECK_FALSE(actual.ephemeral_ports.unavailable_reason.empty()); + } +#if defined(__linux__) + DOCTEST_CHECK(actual.time_wait_scope == "network_namespace_all_processes"); +#else + DOCTEST_CHECK_FALSE(actual.tcp_time_wait.value); + DOCTEST_CHECK_FALSE(actual.tcp_time_wait.unavailable_reason.empty()); +#endif +} + +DOCTEST_TEST_CASE("Connect failures retain their first pre-cleanup snapshot and error identity") { + connection_load::connect_failure_observer observer; + std::ostringstream empty; + observer.write_fields(empty); + DOCTEST_CHECK(empty.str() == ",\"connect_failure_resources\":null"); + std::ostringstream no_event; + observer.write_event(no_event, "client", 42, 7); + DOCTEST_CHECK(no_event.str().empty()); + unsigned queries = 0; + observer.capture(49, "system", 1312, [&] { + ++queries; + auto resources = connection_load::read_port_resources(); + resources.ephemeral_ports = {connection_load::ephemeral_port_range{49152, 65535}, {}}; + resources.tcp_time_wait = {0, {}}; + return resources; + }); + observer.capture(24, "later", 0, [&]() -> connection_load::port_resources { + ++queries; + throw std::runtime_error("must not query twice"); + }); + DOCTEST_CHECK(queries == 1); + std::ostringstream text; + observer.write_fields(text); + DOCTEST_CHECK(text.str().find("\"phase\":\"first_connect_failure_before_cleanup\",\"established\":1312") != + std::string::npos); + DOCTEST_CHECK(text.str().find("\"error_code\":49,\"error_category\":\"system\"") != std::string::npos); + DOCTEST_CHECK(text.str().find("\"ephemeral_ports\":{\"first\":49152,\"last\":65535}") != std::string::npos); + DOCTEST_CHECK(text.str().find("\"tcp_time_wait\":0") != std::string::npos); + DOCTEST_CHECK(text.str().find("free") == std::string::npos); + std::ostringstream event; + observer.write_event(event, "client", 42, 7, true); + DOCTEST_CHECK(event.str().starts_with( + "{\"event\":\"connect_resource_failure\",\"role\":\"client\",\"run_id\":\"42\",\"client_id\":\"7\",")); + DOCTEST_CHECK(event.str().find("\"established\":1312") != std::string::npos); + DOCTEST_CHECK(event.str().ends_with("}\n")); + connection_load::connect_failure_observer failed; + failed.capture(49, "system", 0, []() -> connection_load::port_resources { throw std::runtime_error("query"); }); + std::ostringstream unavailable; + failed.write_fields(unavailable); + DOCTEST_CHECK(unavailable.str().find("Failure resource snapshot could not be captured") != std::string::npos); +} + DOCTEST_TEST_CASE("Connection resource JSON preserves numbers, real zero and explicit missing values") { connection_load::resource_snapshot sample; sample.elapsed_seconds = 2.5;