From 338e1d184b9a1ca4acfc8ebcb7f3740f23194fd2 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 1 Aug 2026 04:04:09 +0800 Subject: [PATCH 01/15] =?UTF-8?q?=E2=9C=A8=20feat:=20define=20bounded=20CP?= =?UTF-8?q?U=20sampling=20evidence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cpu-sample-inventory-result.schema.json | 437 ++++++++++++++++++ schemas/cpu-sampling-spec.schema.json | 104 +++++ schemas/cpu-sampling-validate.schema.json | 210 +++++++++ xprobe/protocol/src/cpu_sampling.rs | 160 +++++++ xprobe/protocol/src/lib.rs | 7 + xprobe/protocol/src/schema.rs | 21 +- xprobe/protocol/tests/contracts.rs | 110 ++++- 7 files changed, 1042 insertions(+), 7 deletions(-) create mode 100644 schemas/cpu-sample-inventory-result.schema.json create mode 100644 schemas/cpu-sampling-spec.schema.json create mode 100644 schemas/cpu-sampling-validate.schema.json create mode 100644 xprobe/protocol/src/cpu_sampling.rs diff --git a/schemas/cpu-sample-inventory-result.schema.json b/schemas/cpu-sample-inventory-result.schema.json new file mode 100644 index 0000000..d874705 --- /dev/null +++ b/schemas/cpu-sample-inventory-result.schema.json @@ -0,0 +1,437 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "CpuSampleInventoryResult", + "type": "object", + "properties": { + "collection": { + "$ref": "#/$defs/CpuSampleCollectionSummary" + }, + "inventory": { + "$ref": "#/$defs/CpuSampleInventory" + }, + "ok": { + "type": "boolean" + }, + "schema_version": { + "$ref": "#/$defs/SchemaVersion" + }, + "session_id": { + "type": "string" + }, + "status": { + "$ref": "#/$defs/SessionStatus" + }, + "symbolization": { + "$ref": "#/$defs/CpuSymbolizationSummary" + }, + "target": { + "$ref": "#/$defs/TargetIdentity" + }, + "warnings": { + "type": "array", + "default": [], + "items": { + "$ref": "#/$defs/Warning" + } + } + }, + "additionalProperties": false, + "required": [ + "schema_version", + "ok", + "session_id", + "status", + "target", + "inventory", + "collection", + "symbolization" + ], + "$defs": { + "CaptureCompleteness": { + "type": "string", + "enum": [ + "complete" + ] + }, + "CpuFrameLanguage": { + "type": "string", + "enum": [ + "native", + "python", + "unknown" + ] + }, + "CpuHotspot": { + "type": "object", + "properties": { + "entry_selector_hint": { + "type": [ + "string", + "null" + ] + }, + "exclusive_proportion": { + "type": "number", + "format": "double" + }, + "exclusive_samples": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "frame": { + "$ref": "#/$defs/CpuStackFrame" + }, + "inclusive_proportion": { + "type": "number", + "format": "double" + }, + "inclusive_samples": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "return_selector_hint": { + "type": [ + "string", + "null" + ] + } + }, + "additionalProperties": false, + "required": [ + "frame", + "inclusive_samples", + "exclusive_samples", + "inclusive_proportion", + "exclusive_proportion" + ] + }, + "CpuSampleCollectionSummary": { + "type": "object", + "properties": { + "completeness": { + "$ref": "#/$defs/CaptureCompleteness" + }, + "group_capacity": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "grouped_samples": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "groups": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "lost_samples": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "observed_samples": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "sample_capacity": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "stack_depth": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "table_utilization": { + "type": "number", + "format": "double" + }, + "thread_capacity": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "threads_attached": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "threads_observed": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "truncated_stacks": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "completeness", + "observed_samples", + "grouped_samples", + "lost_samples", + "sample_capacity", + "group_capacity", + "groups", + "table_utilization", + "stack_depth", + "truncated_stacks", + "threads_observed", + "threads_attached", + "thread_capacity" + ] + }, + "CpuSampleEvent": { + "type": "string", + "enum": [ + "cpu_clock" + ] + }, + "CpuSampleInventory": { + "type": "object", + "properties": { + "duration_ms": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "frequency_hz": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "hotspots": { + "type": "array", + "items": { + "$ref": "#/$defs/CpuHotspot" + } + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "sample_event": { + "$ref": "#/$defs/CpuSampleEvent" + }, + "stack_groups": { + "type": "array", + "items": { + "$ref": "#/$defs/CpuStackGroup" + } + } + }, + "additionalProperties": false, + "required": [ + "sample_event", + "frequency_hz", + "duration_ms", + "stack_groups", + "hotspots" + ] + }, + "CpuStackFrame": { + "type": "object", + "properties": { + "address": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "build_id": { + "type": [ + "string", + "null" + ] + }, + "file_offset": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "language": { + "$ref": "#/$defs/CpuFrameLanguage" + }, + "line": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, + "module_path": { + "type": [ + "string", + "null" + ] + }, + "source_path": { + "type": [ + "string", + "null" + ] + }, + "symbol": { + "type": [ + "string", + "null" + ] + }, + "symbol_offset": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "address", + "language" + ] + }, + "CpuStackGroup": { + "type": "object", + "properties": { + "frames": { + "type": "array", + "items": { + "$ref": "#/$defs/CpuStackFrame" + } + }, + "proportion": { + "type": "number", + "format": "double" + }, + "samples": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "frames", + "samples", + "proportion" + ] + }, + "CpuSymbolizationSummary": { + "type": "object", + "properties": { + "python_status": { + "$ref": "#/$defs/PythonSymbolizationStatus" + }, + "resolved_native_frames": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "resolved_python_frames": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "total_frames": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "unresolved_frames": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "total_frames", + "resolved_native_frames", + "resolved_python_frames", + "unresolved_frames", + "python_status" + ] + }, + "PythonSymbolizationStatus": { + "type": "string", + "enum": [ + "not_detected", + "inactive", + "active", + "unsupported" + ] + }, + "SchemaVersion": { + "type": "string", + "enum": [ + "2.0" + ] + }, + "SessionStatus": { + "type": "string", + "enum": [ + "completed", + "timed_out", + "cancelled", + "failed" + ] + }, + "TargetIdentity": { + "type": "object", + "properties": { + "pid": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "process_start_time": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "pid", + "process_start_time" + ] + }, + "Warning": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "details": { + "type": "object", + "additionalProperties": true, + "default": {} + }, + "message": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "code", + "message" + ] + } + } +} diff --git a/schemas/cpu-sampling-spec.schema.json b/schemas/cpu-sampling-spec.schema.json new file mode 100644 index 0000000..fdeea0c --- /dev/null +++ b/schemas/cpu-sampling-spec.schema.json @@ -0,0 +1,104 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "CpuSamplingSpec", + "type": "object", + "properties": { + "duration_ms": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "frequency_hz": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "max_groups": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "max_samples": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "max_threads": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "sample_event": { + "$ref": "#/$defs/CpuSampleEvent" + }, + "schema_version": { + "$ref": "#/$defs/SchemaVersion" + }, + "stack_depth": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "target": { + "$ref": "#/$defs/TargetIdentity" + }, + "timeout_ms": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "schema_version", + "target", + "sample_event", + "frequency_hz", + "duration_ms", + "timeout_ms", + "max_samples", + "max_groups", + "stack_depth", + "max_threads" + ], + "$defs": { + "CpuSampleEvent": { + "type": "string", + "enum": [ + "cpu_clock" + ] + }, + "SchemaVersion": { + "type": "string", + "enum": [ + "2.0" + ] + }, + "TargetIdentity": { + "type": "object", + "properties": { + "pid": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "process_start_time": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "pid", + "process_start_time" + ] + } + } +} diff --git a/schemas/cpu-sampling-validate.schema.json b/schemas/cpu-sampling-validate.schema.json new file mode 100644 index 0000000..3891e54 --- /dev/null +++ b/schemas/cpu-sampling-validate.schema.json @@ -0,0 +1,210 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "CpuSamplingValidationResult", + "type": "object", + "properties": { + "ebpf": { + "$ref": "#/$defs/CheckResult" + }, + "issues": { + "type": "array", + "default": [], + "items": { + "$ref": "#/$defs/ValidationIssue" + } + }, + "ok": { + "type": "boolean" + }, + "perf_event": { + "$ref": "#/$defs/CheckResult" + }, + "python_status": { + "$ref": "#/$defs/PythonSymbolizationStatus" + }, + "requirements": { + "$ref": "#/$defs/CpuSamplingRequirements" + }, + "schema_version": { + "$ref": "#/$defs/SchemaVersion" + }, + "target": { + "$ref": "#/$defs/TargetIdentity" + }, + "target_threads": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "valid": { + "type": "boolean" + }, + "warnings": { + "type": "array", + "default": [], + "items": { + "$ref": "#/$defs/Warning" + } + } + }, + "additionalProperties": false, + "required": [ + "schema_version", + "ok", + "valid", + "target", + "target_threads", + "requirements", + "perf_event", + "ebpf", + "python_status" + ], + "$defs": { + "CheckResult": { + "type": "object", + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "status": { + "$ref": "#/$defs/CheckStatus" + } + }, + "additionalProperties": false, + "required": [ + "status" + ] + }, + "CheckStatus": { + "type": "string", + "enum": [ + "available", + "restricted", + "unavailable", + "unknown" + ] + }, + "CpuSamplingRequirements": { + "type": "object", + "properties": { + "needs_ebpf": { + "type": "boolean" + }, + "needs_perf_event": { + "type": "boolean" + }, + "target_mutation": { + "type": "boolean" + } + }, + "additionalProperties": false, + "required": [ + "needs_perf_event", + "needs_ebpf", + "target_mutation" + ] + }, + "ErrorCode": { + "type": "string", + "enum": [ + "PERMISSION_DENIED", + "TARGET_NOT_FOUND", + "TARGET_EXITED", + "TARGET_REUSED", + "AMBIGUOUS_TARGET", + "SYMBOL_NOT_FOUND", + "BINARY_NOT_MAPPED", + "INVALID_EVENT_SELECTOR", + "INVALID_CORRELATION_POLICY", + "CUPTI_NOT_AVAILABLE", + "CUPTI_AGENT_NOT_LOADED", + "CUDA_CONTEXT_NOT_FOUND", + "UNSUPPORTED_CUDA_VERSION", + "EVENT_RATE_TOO_HIGH", + "SESSION_LIMIT_EXCEEDED", + "NO_MATCHED_SAMPLES", + "HIGH_UNMATCHED_RATE", + "EVENTS_DROPPED", + "CLOCK_ALIGNMENT_FAILED", + "TRACE_EXPORT_FAILED", + "CLEANUP_FAILED", + "INTERNAL" + ] + }, + "PythonSymbolizationStatus": { + "type": "string", + "enum": [ + "not_detected", + "inactive", + "active", + "unsupported" + ] + }, + "SchemaVersion": { + "type": "string", + "enum": [ + "2.0" + ] + }, + "TargetIdentity": { + "type": "object", + "properties": { + "pid": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "process_start_time": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "pid", + "process_start_time" + ] + }, + "ValidationIssue": { + "type": "object", + "properties": { + "code": { + "$ref": "#/$defs/ErrorCode" + }, + "message": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "code", + "message" + ] + }, + "Warning": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "details": { + "type": "object", + "additionalProperties": true, + "default": {} + }, + "message": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "code", + "message" + ] + } + } +} diff --git a/xprobe/protocol/src/cpu_sampling.rs b/xprobe/protocol/src/cpu_sampling.rs new file mode 100644 index 0000000..a495bbd --- /dev/null +++ b/xprobe/protocol/src/cpu_sampling.rs @@ -0,0 +1,160 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::{ + CaptureCompleteness, CheckResult, SchemaVersion, SessionStatus, TargetIdentity, + ValidationIssue, Warning, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CpuSampleEvent { + CpuClock, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CpuSamplingSpec { + pub schema_version: SchemaVersion, + pub name: Option, + pub target: TargetIdentity, + pub sample_event: CpuSampleEvent, + pub frequency_hz: u64, + pub duration_ms: u64, + pub timeout_ms: u64, + pub max_samples: u64, + pub max_groups: u64, + pub stack_depth: u32, + pub max_threads: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CpuFrameLanguage { + Native, + Python, + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CpuStackFrame { + pub address: u64, + pub module_path: Option, + pub build_id: Option, + pub file_offset: Option, + pub symbol: Option, + pub symbol_offset: Option, + pub language: CpuFrameLanguage, + pub source_path: Option, + pub line: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CpuStackGroup { + pub frames: Vec, + pub samples: u64, + pub proportion: f64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CpuHotspot { + pub frame: CpuStackFrame, + pub inclusive_samples: u64, + pub exclusive_samples: u64, + pub inclusive_proportion: f64, + pub exclusive_proportion: f64, + pub entry_selector_hint: Option, + pub return_selector_hint: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CpuSampleInventory { + pub name: Option, + pub sample_event: CpuSampleEvent, + pub frequency_hz: u64, + pub duration_ms: u64, + pub stack_groups: Vec, + pub hotspots: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum PythonSymbolizationStatus { + NotDetected, + Inactive, + Active, + Unsupported, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CpuSampleCollectionSummary { + pub completeness: CaptureCompleteness, + pub observed_samples: u64, + pub grouped_samples: u64, + pub lost_samples: u64, + pub sample_capacity: u64, + pub group_capacity: u64, + pub groups: u64, + pub table_utilization: f64, + pub stack_depth: u32, + pub truncated_stacks: u64, + pub threads_observed: u32, + pub threads_attached: u32, + pub thread_capacity: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CpuSymbolizationSummary { + pub total_frames: u64, + pub resolved_native_frames: u64, + pub resolved_python_frames: u64, + pub unresolved_frames: u64, + pub python_status: PythonSymbolizationStatus, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CpuSampleInventoryResult { + pub schema_version: SchemaVersion, + pub ok: bool, + pub session_id: String, + pub status: SessionStatus, + pub target: TargetIdentity, + pub inventory: CpuSampleInventory, + pub collection: CpuSampleCollectionSummary, + pub symbolization: CpuSymbolizationSummary, + #[serde(default)] + pub warnings: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CpuSamplingRequirements { + pub needs_perf_event: bool, + pub needs_ebpf: bool, + pub target_mutation: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CpuSamplingValidationResult { + pub schema_version: SchemaVersion, + pub ok: bool, + pub valid: bool, + pub target: TargetIdentity, + pub target_threads: u32, + pub requirements: CpuSamplingRequirements, + pub perf_event: CheckResult, + pub ebpf: CheckResult, + pub python_status: PythonSymbolizationStatus, + #[serde(default)] + pub issues: Vec, + #[serde(default)] + pub warnings: Vec, +} diff --git a/xprobe/protocol/src/lib.rs b/xprobe/protocol/src/lib.rs index 963155d..0e74690 100644 --- a/xprobe/protocol/src/lib.rs +++ b/xprobe/protocol/src/lib.rs @@ -2,6 +2,7 @@ mod capability; mod capture; +mod cpu_sampling; mod discover; mod error; mod event; @@ -17,6 +18,12 @@ pub use capability::{ Capabilities, CapabilityReport, CheckResult, CheckStatus, Environment, SystemChecks, Warning, }; pub use capture::HostCaptureResult; +pub use cpu_sampling::{ + CpuFrameLanguage, CpuHotspot, CpuSampleCollectionSummary, CpuSampleEvent, CpuSampleInventory, + CpuSampleInventoryResult, CpuSamplingRequirements, CpuSamplingSpec, + CpuSamplingValidationResult, CpuStackFrame, CpuStackGroup, CpuSymbolizationSummary, + PythonSymbolizationStatus, +}; pub use discover::{CudaProcessCandidate, DiscoveryResult}; pub use error::{ErrorCode, ErrorResponse, XprobeError}; pub use event::{ diff --git a/xprobe/protocol/src/schema.rs b/xprobe/protocol/src/schema.rs index a14969b..e6f96ac 100644 --- a/xprobe/protocol/src/schema.rs +++ b/xprobe/protocol/src/schema.rs @@ -1,13 +1,14 @@ use schemars::{Schema, schema_for}; use crate::{ - AggregateInventoryResult, CapabilityReport, DiscoveryResult, ErrorResponse, Event, - HostCaptureResult, MeasurementResult, MeasurementSpec, ProcessReport, ResolvedProbe, - TraceExportResult, ValidationResult, + AggregateInventoryResult, CapabilityReport, CpuSampleInventoryResult, CpuSamplingSpec, + CpuSamplingValidationResult, DiscoveryResult, ErrorResponse, Event, HostCaptureResult, + MeasurementResult, MeasurementSpec, ProcessReport, ResolvedProbe, TraceExportResult, + ValidationResult, }; #[must_use] -pub fn generated_schemas() -> [(&'static str, Schema); 12] { +pub fn generated_schemas() -> [(&'static str, Schema); 15] { [ ("event.schema.json", schema_for!(Event)), ("error.schema.json", schema_for!(ErrorResponse)), @@ -20,6 +21,18 @@ pub fn generated_schemas() -> [(&'static str, Schema); 12] { "aggregate-inventory-result.schema.json", schema_for!(AggregateInventoryResult), ), + ( + "cpu-sampling-spec.schema.json", + schema_for!(CpuSamplingSpec), + ), + ( + "cpu-sample-inventory-result.schema.json", + schema_for!(CpuSampleInventoryResult), + ), + ( + "cpu-sampling-validate.schema.json", + schema_for!(CpuSamplingValidationResult), + ), ("capability.schema.json", schema_for!(CapabilityReport)), ("discover.schema.json", schema_for!(DiscoveryResult)), ("inspect.schema.json", schema_for!(ProcessReport)), diff --git a/xprobe/protocol/tests/contracts.rs b/xprobe/protocol/tests/contracts.rs index 3b80afb..afd7259 100644 --- a/xprobe/protocol/tests/contracts.rs +++ b/xprobe/protocol/tests/contracts.rs @@ -3,9 +3,10 @@ use std::{fs, path::PathBuf}; use serde::{Serialize, de::DeserializeOwned}; use serde_json::{Value, json}; use xprobe_protocol::{ - AggregateInventoryResult, CapabilityReport, DiscoveryResult, ErrorResponse, Event, - HostCaptureResult, MeasurementResult, MeasurementSpec, ProcessReport, ResolvedProbe, - TraceExportResult, ValidationResult, schema::generated_schemas, + AggregateInventoryResult, CapabilityReport, CpuSampleInventoryResult, CpuSamplingSpec, + CpuSamplingValidationResult, DiscoveryResult, ErrorResponse, Event, HostCaptureResult, + MeasurementResult, MeasurementSpec, ProcessReport, ResolvedProbe, TraceExportResult, + ValidationResult, schema::generated_schemas, }; fn assert_round_trip(fixture: &Value) @@ -226,6 +227,109 @@ fn aggregate_inventory_contract_round_trips() { })); } +#[test] +fn cpu_sampling_spec_contract_round_trips() { + assert_round_trip::(&json!({ + "schema_version": "2.0", + "name": "cpu_inventory", + "target": {"pid": 1234, "process_start_time": 42}, + "sample_event": "cpu_clock", + "frequency_hz": 99, + "duration_ms": 1000, + "timeout_ms": 30000, + "max_samples": 10000, + "max_groups": 4096, + "stack_depth": 64, + "max_threads": 1024 + })); +} + +#[test] +fn cpu_sample_inventory_contract_round_trips() { + let frame = json!({ + "address": 4198400, + "module_path": "/srv/app", + "build_id": "abcd", + "file_offset": 4096, + "symbol": "handle_request", + "symbol_offset": 0, + "language": "native", + "source_path": null, + "line": null + }); + assert_round_trip::(&json!({ + "schema_version": "2.0", + "ok": true, + "session_id": "xp_cpu_inventory", + "status": "completed", + "target": {"pid": 1234, "process_start_time": 42}, + "inventory": { + "name": "cpu_inventory", + "sample_event": "cpu_clock", + "frequency_hz": 99, + "duration_ms": 1000, + "stack_groups": [{ + "frames": [frame.clone()], + "samples": 80, + "proportion": 0.8 + }], + "hotspots": [{ + "frame": frame, + "inclusive_samples": 80, + "exclusive_samples": 60, + "inclusive_proportion": 0.8, + "exclusive_proportion": 0.6, + "entry_selector_hint": "uprobe:/srv/app:handle_request:entry", + "return_selector_hint": "uprobe:/srv/app:handle_request:return" + }] + }, + "collection": { + "completeness": "complete", + "observed_samples": 100, + "grouped_samples": 100, + "lost_samples": 0, + "sample_capacity": 10000, + "group_capacity": 4096, + "groups": 4, + "table_utilization": 0.0009765625, + "stack_depth": 64, + "truncated_stacks": 0, + "threads_observed": 8, + "threads_attached": 8, + "thread_capacity": 1024 + }, + "symbolization": { + "total_frames": 20, + "resolved_native_frames": 18, + "resolved_python_frames": 0, + "unresolved_frames": 2, + "python_status": "not_detected" + }, + "warnings": [] + })); +} + +#[test] +fn cpu_sampling_validation_contract_round_trips() { + assert_round_trip::(&json!({ + "schema_version": "2.0", + "ok": true, + "valid": true, + "target": {"pid": 1234, "process_start_time": 42}, + "target_threads": 8, + "requirements": { + "needs_perf_event": true, + "needs_ebpf": true, + "target_mutation": false + }, + "perf_event": {"status": "available", "detail": "perf_event_paranoid=1"}, + "ebpf": {"status": "available", "detail": "CAP_BPF and CAP_PERFMON"}, + "python_status": "inactive", + "issues": [], + "warnings": [] + })); +} + #[test] fn trace_export_contract_round_trips() { assert_round_trip::(&json!({ From 2caa9020cf4bfbdaa64da4d0a7147b6f879d8abd Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 1 Aug 2026 04:13:03 +0800 Subject: [PATCH 02/15] =?UTF-8?q?=E2=9C=A8=20feat:=20validate=20bounded=20?= =?UTF-8?q?CPU=20sampling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- schemas/cpu-sampling-validate.schema.json | 8 - xprobe/cli/src/main.rs | 86 ++++-- xprobe/cli/tests/validate.rs | 35 ++- xprobe/core/src/cpu_sampling.rs | 335 ++++++++++++++++++++++ xprobe/core/src/lib.rs | 1 + xprobe/protocol/src/cpu_sampling.rs | 2 - xprobe/protocol/tests/contracts.rs | 6 +- 7 files changed, 436 insertions(+), 37 deletions(-) create mode 100644 xprobe/core/src/cpu_sampling.rs diff --git a/schemas/cpu-sampling-validate.schema.json b/schemas/cpu-sampling-validate.schema.json index 3891e54..1590213 100644 --- a/schemas/cpu-sampling-validate.schema.json +++ b/schemas/cpu-sampling-validate.schema.json @@ -3,9 +3,6 @@ "title": "CpuSamplingValidationResult", "type": "object", "properties": { - "ebpf": { - "$ref": "#/$defs/CheckResult" - }, "issues": { "type": "array", "default": [], @@ -56,7 +53,6 @@ "target_threads", "requirements", "perf_event", - "ebpf", "python_status" ], "$defs": { @@ -90,9 +86,6 @@ "CpuSamplingRequirements": { "type": "object", "properties": { - "needs_ebpf": { - "type": "boolean" - }, "needs_perf_event": { "type": "boolean" }, @@ -103,7 +96,6 @@ "additionalProperties": false, "required": [ "needs_perf_event", - "needs_ebpf", "target_mutation" ] }, diff --git a/xprobe/cli/src/main.rs b/xprobe/cli/src/main.rs index 85a5cb6..7e8963f 100644 --- a/xprobe/cli/src/main.rs +++ b/xprobe/cli/src/main.rs @@ -22,17 +22,19 @@ use xprobe_collector::{ linux::{self, LinuxCaptureRequest}, uprobe::{self, UprobeRequest}, }; -use xprobe_core::{cupti_compat, discover, doctor, inject, inspect, resolve, validate}; +use xprobe_core::{ + cpu_sampling, cupti_compat, discover, doctor, inject, inspect, resolve, validate, +}; use xprobe_correlator::{MeasureError, MeasureOptions, measure}; use xprobe_exporter::{events_to_chrome_trace, events_to_jsonl}; use xprobe_protocol::{ AggregateActivity, AggregateCollectionSummary, AggregateDuration, AggregateGroup, AggregateInventory, AggregateInventoryResult, CapabilityReport, CaptureCompleteness, - CheckResult, ClockDomain, CuptiCollectionSummary, DiscoveryResult, ErrorCode, ErrorResponse, - Event, EventSource, EventType, ExportFormat, HostCaptureResult, MatchPolicy, MeasurementMode, - MeasurementResult, MeasurementSpec, MemcpyKind, ProcessReport, ResolvedCudaSelector, - ResolvedLinuxSelector, ResolvedProbe, SchemaVersion, SessionStatus, TargetIdentity, - TraceExportResult, ValidationResult, Warning, XprobeError, + CheckResult, ClockDomain, CpuSamplingValidationResult, CuptiCollectionSummary, DiscoveryResult, + ErrorCode, ErrorResponse, Event, EventSource, EventType, ExportFormat, HostCaptureResult, + MatchPolicy, MeasurementMode, MeasurementResult, MeasurementSpec, MemcpyKind, ProcessReport, + ResolvedCudaSelector, ResolvedLinuxSelector, ResolvedProbe, SchemaVersion, SessionStatus, + TargetIdentity, TraceExportResult, ValidationResult, Warning, XprobeError, }; #[derive(Debug, Parser)] @@ -230,27 +232,22 @@ struct ValidateArgs { /// Start event selector. #[arg(long)] - from: String, + from: Option, /// End event selector. #[arg(long)] - to: String, + to: Option, /// Correlation policy. #[arg(long = "match")] - match_policy: String, - - /// Emit only the versioned JSON result on stdout. - #[arg(long)] - json: bool, + match_policy: Option, - /// Disable colored output. - #[arg(long)] - no_color: bool, + /// Validate bounded PID-scoped CPU sampling instead of an event pair. + #[arg(long, conflicts_with_all = ["from", "to", "match_policy"])] + cpu_sample: bool, - /// Never wait for user input. - #[arg(long)] - non_interactive: bool, + #[command(flatten)] + output: CommonOutputArgs, } #[derive(Debug, Args)] @@ -3021,9 +3018,13 @@ fn run_validate(args: ValidateArgs) -> ExitCode { from, to, match_policy, - json, - no_color: _, - non_interactive: _, + cpu_sample, + output: + CommonOutputArgs { + json, + no_color: _, + non_interactive: _, + }, } = args; let report = match inspect::run(pid) { Ok(report) => report, @@ -3032,6 +3033,21 @@ fn run_validate(args: ValidateArgs) -> ExitCode { } }; + if cpu_sample { + return match cpu_sampling::validate(&report) { + Ok(result) => emit_cpu_sampling_validation(&result, json), + Err(error) => emit_error(error.code(), error.to_string(), error.recoverable(), json), + }; + } + let (Some(from), Some(to), Some(match_policy)) = (from, to, match_policy) else { + return emit_error( + ErrorCode::InvalidEventSelector, + "validate requires --from, --to, and --match unless --cpu-sample is used".to_owned(), + true, + json, + ); + }; + match validate::run(&report, &from, &to, &match_policy) { Ok(result) => { if json { @@ -3049,6 +3065,32 @@ fn run_validate(args: ValidateArgs) -> ExitCode { } } +fn emit_cpu_sampling_validation(result: &CpuSamplingValidationResult, json: bool) -> ExitCode { + if json { + println!( + "{}", + serde_json::to_string_pretty(result) + .expect("CPU sampling validation result must serialize") + ); + } else { + println!( + "CPU sampling validation: {}", + if result.valid { "valid" } else { "invalid" } + ); + println!("Target: PID {}", result.target.pid); + println!("Threads: {}", result.target_threads); + println!("Python symbols: {:?}", result.python_status); + print_check("perf event", &result.perf_event); + for issue in &result.issues { + println!("Issue: {}: {}", issue.code, issue.message); + } + for warning in &result.warnings { + println!("Warning: {}: {}", warning.code, warning.message); + } + } + ExitCode::SUCCESS +} + fn run_resolve(args: ResolveArgs) -> ExitCode { let ResolveArgs { pid, diff --git a/xprobe/cli/tests/validate.rs b/xprobe/cli/tests/validate.rs index 3b693fe..675718a 100644 --- a/xprobe/cli/tests/validate.rs +++ b/xprobe/cli/tests/validate.rs @@ -1,6 +1,9 @@ use std::process::Command; -use xprobe_protocol::{AgentActivation, ErrorCode, ErrorResponse, MatchPolicy, ValidationResult}; +use xprobe_protocol::{ + AgentActivation, CpuSamplingValidationResult, ErrorCode, ErrorResponse, MatchPolicy, + PythonSymbolizationStatus, ValidationResult, +}; #[test] fn validate_reports_environment_requirements_without_attaching() { @@ -80,3 +83,33 @@ fn validate_rejects_an_invalid_kernel_regex() { serde_json::from_slice(&output.stdout).expect("stdout must contain error JSON"); assert_eq!(error.error.code, ErrorCode::InvalidEventSelector); } + +#[test] +fn validate_accepts_pid_scoped_cpu_sampling_without_ebpf() { + let output = Command::new(env!("CARGO_BIN_EXE_xprobe")) + .args([ + "validate", + "--pid", + &std::process::id().to_string(), + "--cpu-sample", + "--json", + "--non-interactive", + "--no-color", + ]) + .output() + .expect("xprobe validate must run"); + + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.stderr.is_empty()); + let result: CpuSamplingValidationResult = + serde_json::from_slice(&output.stdout).expect("stdout must contain validation JSON"); + assert!(result.valid); + assert!(result.requirements.needs_perf_event); + assert!(!result.requirements.target_mutation); + assert!(result.target_threads >= 1); + assert_eq!(result.python_status, PythonSymbolizationStatus::NotDetected); +} diff --git a/xprobe/core/src/cpu_sampling.rs b/xprobe/core/src/cpu_sampling.rs new file mode 100644 index 0000000..b2f2b61 --- /dev/null +++ b/xprobe/core/src/cpu_sampling.rs @@ -0,0 +1,335 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + error::Error, + ffi::OsStr, + fmt, fs, io, + path::{Path, PathBuf}, +}; + +use object::{Object, ObjectSymbol}; +use xprobe_protocol::{ + CheckResult, CheckStatus, CpuSamplingRequirements, CpuSamplingValidationResult, ErrorCode, + ProcessReport, PythonSymbolizationStatus, SchemaVersion, ValidationIssue, Warning, +}; + +use crate::{ + doctor::{self, DoctorError}, + inspect::{self, InspectError}, +}; + +const MAX_PERF_MAP_BYTES: u64 = 16 * 1024 * 1024; + +#[derive(Debug)] +pub enum CpuSamplingError { + Inspect(InspectError), + Doctor(DoctorError), + Io { path: PathBuf, source: io::Error }, + InvalidElf { path: PathBuf, reason: String }, + InvalidTask { path: PathBuf, value: String }, + Limit { name: &'static str, value: u64 }, +} + +impl CpuSamplingError { + #[must_use] + pub fn code(&self) -> ErrorCode { + match self { + Self::Inspect(error) => error.code(), + Self::Io { source, .. } if source.kind() == io::ErrorKind::PermissionDenied => { + ErrorCode::PermissionDenied + } + Self::Limit { .. } => ErrorCode::SessionLimitExceeded, + Self::Doctor(_) + | Self::Io { .. } + | Self::InvalidElf { .. } + | Self::InvalidTask { .. } => ErrorCode::Internal, + } + } + + #[must_use] + pub fn recoverable(&self) -> bool { + match self { + Self::Inspect(error) => error.recoverable(), + Self::Io { source, .. } => { + matches!( + source.kind(), + io::ErrorKind::PermissionDenied | io::ErrorKind::NotFound + ) + } + Self::Limit { .. } => true, + Self::Doctor(_) | Self::InvalidElf { .. } | Self::InvalidTask { .. } => false, + } + } +} + +impl fmt::Display for CpuSamplingError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Inspect(error) => error.fmt(formatter), + Self::Doctor(error) => { + write!(formatter, "CPU sampling capability check failed: {error}") + } + Self::Io { path, source } => { + write!(formatter, "failed to read {}: {source}", path.display()) + } + Self::InvalidElf { path, reason } => { + write!( + formatter, + "failed to parse ELF {}: {reason}", + path.display() + ) + } + Self::InvalidTask { path, value } => { + write!( + formatter, + "invalid task entry {value:?} in {}", + path.display() + ) + } + Self::Limit { name, value } => { + write!( + formatter, + "CPU sampling {name} exceeds the supported bound: {value}" + ) + } + } + } +} + +impl Error for CpuSamplingError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Inspect(error) => Some(error), + Self::Doctor(error) => Some(error), + Self::Io { source, .. } => Some(source), + Self::InvalidElf { .. } | Self::InvalidTask { .. } | Self::Limit { .. } => None, + } + } +} + +impl From for CpuSamplingError { + fn from(error: InspectError) -> Self { + Self::Inspect(error) + } +} + +impl From for CpuSamplingError { + fn from(error: DoctorError) -> Self { + Self::Doctor(error) + } +} + +/// Validate CPU sampling requirements without opening a perf event or loading BPF. +/// +/// # Errors +/// +/// Returns [`CpuSamplingError`] when target identity, procfs, local capability, +/// or mapped ELF inspection fails unexpectedly. +pub fn validate(report: &ProcessReport) -> Result { + inspect::verify_target(&report.target)?; + let target_threads = target_threads(&report.target)?; + let capabilities = doctor::run()?; + let perf_event = effective_perf_check(&capabilities, report.credentials.effective_uid); + let python_status = python_symbolization_status(report)?; + + let mut issues = Vec::new(); + if perf_event.status != CheckStatus::Available { + issues.push(ValidationIssue { + code: ErrorCode::PermissionDenied, + message: "CPU sampling is restricted by the effective perf event policy".to_owned(), + }); + } + + let mut warnings = Vec::new(); + match python_status { + PythonSymbolizationStatus::Inactive => warnings.push(warning( + "PYTHON_PERF_TRAMPOLINE_INACTIVE", + "CPython perf trampoline support is present but inactive; native frames remain available", + )), + PythonSymbolizationStatus::Unsupported => warnings.push(warning( + "PYTHON_PERF_TRAMPOLINE_UNSUPPORTED", + "the mapped CPython build does not expose perf trampoline symbols; native frames remain available", + )), + PythonSymbolizationStatus::NotDetected | PythonSymbolizationStatus::Active => {} + } + + inspect::verify_target(&report.target)?; + Ok(CpuSamplingValidationResult { + schema_version: SchemaVersion::current(), + ok: true, + valid: issues.is_empty(), + target: report.target.clone(), + target_threads, + requirements: CpuSamplingRequirements { + needs_perf_event: true, + target_mutation: false, + }, + perf_event, + python_status, + issues, + warnings, + }) +} + +/// Return the target's current thread IDs after verifying process identity. +/// +/// # Errors +/// +/// Returns [`CpuSamplingError`] for target changes, malformed task entries, I/O +/// failures, or a task count that cannot be represented by the public contract. +pub fn target_thread_ids( + target: &xprobe_protocol::TargetIdentity, +) -> Result, CpuSamplingError> { + inspect::verify_target(target)?; + let path = PathBuf::from(format!("/proc/{}/task", target.pid)); + let entries = fs::read_dir(&path).map_err(|source| CpuSamplingError::Io { + path: path.clone(), + source, + })?; + let mut threads = BTreeSet::new(); + for entry in entries { + let entry = entry.map_err(|source| CpuSamplingError::Io { + path: path.clone(), + source, + })?; + let value = entry.file_name(); + let text = value + .to_str() + .ok_or_else(|| CpuSamplingError::InvalidTask { + path: path.clone(), + value: value.to_string_lossy().into_owned(), + })?; + let tid = text + .parse::() + .map_err(|_| CpuSamplingError::InvalidTask { + path: path.clone(), + value: text.to_owned(), + })?; + threads.insert(tid); + } + inspect::verify_target(target)?; + Ok(threads.into_iter().collect()) +} + +fn target_threads(target: &xprobe_protocol::TargetIdentity) -> Result { + let count = target_thread_ids(target)?.len(); + u32::try_from(count).map_err(|_| CpuSamplingError::Limit { + name: "thread count", + value: u64::try_from(count).unwrap_or(u64::MAX), + }) +} + +fn effective_perf_check( + report: &xprobe_protocol::CapabilityReport, + target_effective_uid: u32, +) -> CheckResult { + let same_user = report.environment.effective_uid == target_effective_uid; + let user_space_process_sampling_allowed = report + .checks + .perf_event_paranoid + .detail + .as_deref() + .and_then(|value| value.parse::().ok()) + .is_some_and(|value| value <= 2); + if report.environment.effective_uid == 0 + || report.capabilities.uprobe + || (same_user && user_space_process_sampling_allowed) + { + return CheckResult { + status: CheckStatus::Available, + detail: Some(format!( + "PID-scoped user-space sampling is permitted; perf_event_paranoid={}", + report + .checks + .perf_event_paranoid + .detail + .as_deref() + .unwrap_or("unknown") + )), + }; + } + report.checks.perf_event_paranoid.clone() +} + +fn python_symbolization_status( + report: &ProcessReport, +) -> Result { + let mut candidates = vec![PathBuf::from(&report.executable)]; + candidates.extend( + report + .loaded_libraries + .iter() + .map(PathBuf::from) + .filter(|path| { + path.file_name() + .and_then(OsStr::to_str) + .is_some_and(|name| name.starts_with("libpython")) + }), + ); + candidates.sort(); + candidates.dedup(); + + let mut python_detected = false; + let mut trampoline_supported = false; + for path in candidates { + let symbols = elf_symbol_names(&path)?; + let is_python = symbols.contains("Py_Initialize") + || symbols.contains("_PyEval_EvalFrameDefault") + || symbols.contains("PyUnstable_PerfMapState_Init"); + if !is_python { + continue; + } + python_detected = true; + trampoline_supported |= symbols.contains("_Py_trampoline_func_start") + && symbols.contains("PyUnstable_WritePerfMapEntry"); + } + if !python_detected { + return Ok(PythonSymbolizationStatus::NotDetected); + } + if !trampoline_supported { + return Ok(PythonSymbolizationStatus::Unsupported); + } + + let path = PathBuf::from(format!( + "/proc/{}/root/tmp/perf-{}.map", + report.target.pid, report.target.pid + )); + match fs::metadata(&path) { + Ok(metadata) if metadata.len() > MAX_PERF_MAP_BYTES => Err(CpuSamplingError::Limit { + name: "perf map bytes", + value: metadata.len(), + }), + Ok(metadata) if metadata.is_file() && metadata.len() != 0 => { + Ok(PythonSymbolizationStatus::Active) + } + Ok(_) => Ok(PythonSymbolizationStatus::Inactive), + Err(source) if source.kind() == io::ErrorKind::NotFound => { + Ok(PythonSymbolizationStatus::Inactive) + } + Err(source) => Err(CpuSamplingError::Io { path, source }), + } +} + +fn elf_symbol_names(path: &Path) -> Result, CpuSamplingError> { + let bytes = fs::read(path).map_err(|source| CpuSamplingError::Io { + path: path.to_owned(), + source, + })?; + let file = + object::File::parse(bytes.as_slice()).map_err(|error| CpuSamplingError::InvalidElf { + path: path.to_owned(), + reason: error.to_string(), + })?; + Ok(file + .symbols() + .chain(file.dynamic_symbols()) + .filter_map(|symbol| symbol.name().ok().map(str::to_owned)) + .collect()) +} + +fn warning(code: &str, message: &str) -> Warning { + Warning { + code: code.to_owned(), + message: message.to_owned(), + details: BTreeMap::new(), + } +} diff --git a/xprobe/core/src/lib.rs b/xprobe/core/src/lib.rs index 2b44cd3..a9ec004 100644 --- a/xprobe/core/src/lib.rs +++ b/xprobe/core/src/lib.rs @@ -1,5 +1,6 @@ //! Measurement orchestration and domain logic for xprobe. +pub mod cpu_sampling; pub mod cupti_compat; pub mod discover; pub mod doctor; diff --git a/xprobe/protocol/src/cpu_sampling.rs b/xprobe/protocol/src/cpu_sampling.rs index a495bbd..a1538bd 100644 --- a/xprobe/protocol/src/cpu_sampling.rs +++ b/xprobe/protocol/src/cpu_sampling.rs @@ -137,7 +137,6 @@ pub struct CpuSampleInventoryResult { #[serde(deny_unknown_fields)] pub struct CpuSamplingRequirements { pub needs_perf_event: bool, - pub needs_ebpf: bool, pub target_mutation: bool, } @@ -151,7 +150,6 @@ pub struct CpuSamplingValidationResult { pub target_threads: u32, pub requirements: CpuSamplingRequirements, pub perf_event: CheckResult, - pub ebpf: CheckResult, pub python_status: PythonSymbolizationStatus, #[serde(default)] pub issues: Vec, diff --git a/xprobe/protocol/tests/contracts.rs b/xprobe/protocol/tests/contracts.rs index afd7259..3b91070 100644 --- a/xprobe/protocol/tests/contracts.rs +++ b/xprobe/protocol/tests/contracts.rs @@ -247,7 +247,7 @@ fn cpu_sampling_spec_contract_round_trips() { #[test] fn cpu_sample_inventory_contract_round_trips() { let frame = json!({ - "address": 4198400, + "address": 4_198_400, "module_path": "/srv/app", "build_id": "abcd", "file_offset": 4096, @@ -291,7 +291,7 @@ fn cpu_sample_inventory_contract_round_trips() { "sample_capacity": 10000, "group_capacity": 4096, "groups": 4, - "table_utilization": 0.0009765625, + "table_utilization": 0.000_976_562_5, "stack_depth": 64, "truncated_stacks": 0, "threads_observed": 8, @@ -319,11 +319,9 @@ fn cpu_sampling_validation_contract_round_trips() { "target_threads": 8, "requirements": { "needs_perf_event": true, - "needs_ebpf": true, "target_mutation": false }, "perf_event": {"status": "available", "detail": "perf_event_paranoid=1"}, - "ebpf": {"status": "available", "detail": "CAP_BPF and CAP_PERFMON"}, "python_status": "inactive", "issues": [], "warnings": [] From 0e390974097270717314d1562b0f137f1ccba537 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 1 Aug 2026 04:32:51 +0800 Subject: [PATCH 03/15] =?UTF-8?q?=E2=9C=A8=20feat:=20collect=20CPU=20and?= =?UTF-8?q?=20Python=20hotspots?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 274 +++++- Cargo.toml | 1 + .../cpu-sample-inventory-result.schema.json | 7 +- xprobe/cli/src/main.rs | 376 +++++++- xprobe/cli/tests/cpu_sampling.rs | 112 +++ xprobe/collector/Cargo.toml | 1 + xprobe/collector/src/cpu_sampling.rs | 421 +++++++++ xprobe/collector/src/lib.rs | 1 + xprobe/core/src/cpu_sampling_analysis.rs | 807 ++++++++++++++++++ xprobe/core/src/lib.rs | 1 + xprobe/protocol/src/cpu_sampling.rs | 14 +- xprobe/protocol/src/lib.rs | 8 +- 12 files changed, 1998 insertions(+), 25 deletions(-) create mode 100644 xprobe/cli/tests/cpu_sampling.rs create mode 100644 xprobe/collector/src/cpu_sampling.rs create mode 100644 xprobe/core/src/cpu_sampling_analysis.rs diff --git a/Cargo.lock b/Cargo.lock index cdcc629..487ae71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,6 +61,38 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn 2.0.117", +] + [[package]] name = "bitflags" version = "2.13.1" @@ -74,7 +106,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" dependencies = [ "find-msvc-tools", - "shlex", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", ] [[package]] @@ -89,6 +130,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.6.1" @@ -150,12 +202,112 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "futures" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-executor" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" + +[[package]] +name = "futures-macro" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + [[package]] name = "heck" version = "0.5.0" @@ -168,6 +320,24 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -203,12 +373,34 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "nix" version = "0.31.3" @@ -221,6 +413,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "object" version = "0.36.7" @@ -236,12 +438,43 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "perf-event-open" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d36cfad4eb4d20694b0a72dd54708eb48801230bcb4534b83615a5898949b46" +dependencies = [ + "anyhow", + "arrayvec", + "bindgen", + "futures", + "itertools 0.14.0", + "libc", + "thiserror", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "pkg-config" version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.117", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -309,6 +542,12 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "schemars" version = "1.2.1" @@ -388,12 +627,24 @@ dependencies = [ "zmij", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "shlex" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "strsim" version = "0.11.1" @@ -422,6 +673,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.0", +] + [[package]] name = "unicode-ident" version = "1.0.24" @@ -477,6 +748,7 @@ name = "xprobe-collector" version = "0.4.1" dependencies = [ "libbpf-rs", + "perf-event-open", "serde_json", "xprobe-protocol", ] diff --git a/Cargo.toml b/Cargo.toml index 393cac2..29ee202 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ cpp_demangle = "0.4" libbpf-rs = { version = "0.26.2", features = ["vendored"] } nix = { version = "0.31.3", features = ["process", "ptrace", "signal", "uio"] } object = { version = "0.36", default-features = false, features = ["read", "std"] } +perf-event-open = { version = "0.6.3", features = ["linux-4.8"] } regex = "1.11" schemars = "1.0" serde = { version = "1.0", features = ["derive"] } diff --git a/schemas/cpu-sample-inventory-result.schema.json b/schemas/cpu-sample-inventory-result.schema.json index d874705..730388f 100644 --- a/schemas/cpu-sample-inventory-result.schema.json +++ b/schemas/cpu-sample-inventory-result.schema.json @@ -47,10 +47,11 @@ "symbolization" ], "$defs": { - "CaptureCompleteness": { + "CpuCaptureCompleteness": { "type": "string", "enum": [ - "complete" + "complete", + "incomplete" ] }, "CpuFrameLanguage": { @@ -111,7 +112,7 @@ "type": "object", "properties": { "completeness": { - "$ref": "#/$defs/CaptureCompleteness" + "$ref": "#/$defs/CpuCaptureCompleteness" }, "group_capacity": { "type": "integer", diff --git a/xprobe/cli/src/main.rs b/xprobe/cli/src/main.rs index 7e8963f..d539d56 100644 --- a/xprobe/cli/src/main.rs +++ b/xprobe/cli/src/main.rs @@ -18,23 +18,28 @@ static EXPORT_SEQUENCE: AtomicU64 = AtomicU64::new(0); use clap::{Args, Parser, Subcommand, ValueEnum}; use xprobe_collector::{ - completed, cupti, + completed, + cpu_sampling::{self as cpu_collector, CpuSamplingRequest}, + cupti, linux::{self, LinuxCaptureRequest}, uprobe::{self, UprobeRequest}, }; use xprobe_core::{ - cpu_sampling, cupti_compat, discover, doctor, inject, inspect, resolve, validate, + cpu_sampling, + cpu_sampling_analysis::{self, CpuCaptureData, RawCpuSample}, + cupti_compat, discover, doctor, inject, inspect, resolve, validate, }; use xprobe_correlator::{MeasureError, MeasureOptions, measure}; use xprobe_exporter::{events_to_chrome_trace, events_to_jsonl}; use xprobe_protocol::{ AggregateActivity, AggregateCollectionSummary, AggregateDuration, AggregateGroup, AggregateInventory, AggregateInventoryResult, CapabilityReport, CaptureCompleteness, - CheckResult, ClockDomain, CpuSamplingValidationResult, CuptiCollectionSummary, DiscoveryResult, - ErrorCode, ErrorResponse, Event, EventSource, EventType, ExportFormat, HostCaptureResult, - MatchPolicy, MeasurementMode, MeasurementResult, MeasurementSpec, MemcpyKind, ProcessReport, - ResolvedCudaSelector, ResolvedLinuxSelector, ResolvedProbe, SchemaVersion, SessionStatus, - TargetIdentity, TraceExportResult, ValidationResult, Warning, XprobeError, + CheckResult, ClockDomain, CpuSampleEvent, CpuSampleInventoryResult, CpuSamplingSpec, + CpuSamplingValidationResult, CuptiCollectionSummary, DiscoveryResult, ErrorCode, ErrorResponse, + Event, EventSource, EventType, ExportFormat, HostCaptureResult, MatchPolicy, MeasurementMode, + MeasurementResult, MeasurementSpec, MemcpyKind, ProcessReport, ResolvedCudaSelector, + ResolvedLinuxSelector, ResolvedProbe, SchemaVersion, SessionStatus, TargetIdentity, + TraceExportResult, ValidationResult, Warning, XprobeError, }; #[derive(Debug, Parser)] @@ -252,8 +257,8 @@ struct ValidateArgs { #[derive(Debug, Args)] struct MeasureArgs { - /// Versioned `MeasurementSpec` JSON for a live target. - #[arg(long, conflicts_with_all = ["input", "pid", "from", "to", "match_policy", "samples", "duration_ms", "timeout_ms", "max_events", "aggregate", "max_groups", "name"])] + /// Versioned measurement JSON for a live target. + #[arg(long, conflicts_with_all = ["input", "pid", "from", "to", "match_policy", "cpu_sample", "samples", "duration_ms", "timeout_ms", "max_events", "max_samples", "frequency_hz", "stack_depth", "max_threads", "aggregate", "max_groups", "name"])] spec: Option, /// Completed CUPTI binary, host capture JSON, or Event JSONL; repeat to merge. @@ -296,6 +301,42 @@ struct MeasureArgs { #[arg(long = "match")] match_policy: Option, + /// Collect a bounded PID-scoped CPU hotspot inventory. + #[arg(long, conflicts_with_all = ["input", "from", "to", "match_policy", "samples", "aggregate", "events_out", "format", "cupti_socket", "agent"])] + cpu_sample: bool, + + /// Target CPU sampling frequency in hertz. + #[arg( + long, + requires = "cpu_sample", + default_value_if("cpu_sample", "true", "99") + )] + frequency_hz: Option, + + /// Bound all observed CPU sample records. + #[arg( + long, + requires = "cpu_sample", + default_value_if("cpu_sample", "true", "10000") + )] + max_samples: Option, + + /// Bound user-space callchain depth. + #[arg( + long, + requires = "cpu_sample", + default_value_if("cpu_sample", "true", "64") + )] + stack_depth: Option, + + /// Bound target threads attached by CPU sampling. + #[arg( + long, + requires = "cpu_sample", + default_value_if("cpu_sample", "true", "1024") + )] + max_threads: Option, + /// Stop after this many matched samples. #[arg(long)] samples: Option, @@ -312,11 +353,11 @@ struct MeasureArgs { #[arg(long, conflicts_with_all = ["input", "samples", "events_out", "format"])] aggregate: bool, - /// Bound distinct activity groups retained by --aggregate. + /// Bound distinct groups retained by aggregate or CPU inventory mode. #[arg( long, - requires = "aggregate", - default_value_if("aggregate", "true", "4096") + default_value_if("aggregate", "true", "4096"), + default_value_if("cpu_sample", "true", "256") )] max_groups: Option, @@ -695,6 +736,7 @@ fn remove_temporary_artifact(path: &Path, original: String) -> String { } } +#[allow(clippy::too_many_lines)] fn run_measure(args: MeasureArgs) -> ExitCode { let MeasureArgs { spec, @@ -708,6 +750,11 @@ fn run_measure(args: MeasureArgs) -> ExitCode { from, to, match_policy, + cpu_sample, + frequency_hz, + max_samples, + stack_depth, + max_threads, samples, duration_ms, max_events, @@ -731,6 +778,45 @@ fn run_measure(args: MeasureArgs) -> ExitCode { json, ); } + if cpu_sample { + let Some(pid) = pid else { + return emit_error( + ErrorCode::InvalidEventSelector, + "--cpu-sample requires --pid".to_owned(), + true, + json, + ); + }; + let Some(duration_ms) = duration_ms else { + return emit_error( + ErrorCode::SessionLimitExceeded, + "--cpu-sample requires --duration-ms".to_owned(), + true, + json, + ); + }; + return run_direct_cpu_sampling( + pid, + name, + frequency_hz.expect("clap supplies CPU sampling frequency"), + duration_ms, + timeout_ms, + max_samples.expect("clap supplies CPU sample capacity"), + u64::try_from(max_groups.expect("clap supplies CPU group capacity")) + .unwrap_or(u64::MAX), + stack_depth.expect("clap supplies CPU stack depth"), + max_threads.expect("clap supplies CPU thread capacity"), + json, + ); + } + if max_groups.is_some() && !aggregate { + return emit_error( + ErrorCode::InvalidEventSelector, + "--max-groups requires --aggregate or --cpu-sample".to_owned(), + true, + json, + ); + } let (from, to, match_policy) = match parse_measure_selection(from, to, match_policy) { Ok(selection) => selection, Err(error) => { @@ -798,6 +884,223 @@ fn run_measure(args: MeasureArgs) -> ExitCode { ) } +#[allow(clippy::too_many_arguments)] +fn run_direct_cpu_sampling( + pid: u32, + name: Option, + frequency_hz: u64, + duration_ms: u64, + timeout_ms: u64, + max_samples: u64, + max_groups: u64, + stack_depth: u32, + max_threads: u32, + json: bool, +) -> ExitCode { + let report = match inspect::run(pid) { + Ok(report) => report, + Err(error) => { + return emit_error(error.code(), error.to_string(), error.recoverable(), json); + } + }; + let spec = CpuSamplingSpec { + schema_version: SchemaVersion::current(), + name, + target: report.target.clone(), + sample_event: CpuSampleEvent::CpuClock, + frequency_hz, + duration_ms, + timeout_ms, + max_samples, + max_groups, + stack_depth, + max_threads, + }; + run_cpu_sampling_spec(&spec, Some(report), json) +} + +fn run_cpu_sampling_spec( + spec: &CpuSamplingSpec, + inspected: Option, + json: bool, +) -> ExitCode { + let report = match inspected.map_or_else(|| inspect::run(spec.target.pid), Ok) { + Ok(report) => report, + Err(error) => { + return emit_error(error.code(), error.to_string(), error.recoverable(), json); + } + }; + if let Err(error) = inspect::verify_target(&spec.target) { + return emit_error(error.code(), error.to_string(), error.recoverable(), json); + } + let validation = match cpu_sampling::validate(&report) { + Ok(validation) => validation, + Err(error) => { + return emit_error(error.code(), error.to_string(), error.recoverable(), json); + } + }; + if !validation.valid { + let message = validation.issues.first().map_or_else( + || "CPU sampling requirements are not satisfied".to_owned(), + |issue| issue.message.clone(), + ); + return emit_error(ErrorCode::PermissionDenied, message, true, json); + } + let request = match cpu_collector_request(spec) { + Ok(request) => request, + Err(error) => return emit_command_failure(error, json), + }; + let capture = match cpu_collector::collect(&request) { + Ok(capture) => capture, + Err(error) => return emit_command_failure(cpu_collector_failure(&error), json), + }; + if capture.observed_samples == 0 { + return emit_error( + ErrorCode::NoMatchedSamples, + "CPU sampling completed without observing an on-CPU sample".to_owned(), + true, + json, + ); + } + let capture = CpuCaptureData { + samples: capture + .stacks + .into_iter() + .map(|stack| RawCpuSample { + thread_id: stack.thread_id, + addresses: stack.addresses, + }) + .collect(), + observed_samples: capture.observed_samples, + lost_samples: capture.lost_samples, + truncated_stacks: capture.truncated_stacks, + throttled_records: capture.throttled_records, + threads_observed: capture.threads_observed, + threads_attached: capture.threads_attached, + skipped_threads: capture.skipped_threads, + capacity_reached: capture.capacity_reached, + python_status: validation.python_status, + }; + let result = match cpu_sampling_analysis::analyze( + &report, + spec, + format!("xp_cpu_{}", std::process::id()), + &capture, + ) { + Ok(result) => result, + Err(error) => { + return emit_error(error.code(), error.to_string(), error.recoverable(), json); + } + }; + emit_cpu_inventory(&result, json) +} + +fn cpu_collector_request(spec: &CpuSamplingSpec) -> Result { + if spec.frequency_hz == 0 + || spec.duration_ms == 0 + || spec.max_samples == 0 + || spec.max_groups == 0 + || spec.stack_depth == 0 + || spec.max_threads == 0 + { + return Err(CommandFailure::new( + ErrorCode::SessionLimitExceeded, + "CPU sampling frequency, duration, samples, groups, stack depth, and threads must be positive", + true, + )); + } + let max_samples = usize::try_from(spec.max_samples).map_err(|error| { + CommandFailure::new( + ErrorCode::SessionLimitExceeded, + format!("CPU max_samples exceed this platform: {error}"), + true, + ) + })?; + let max_threads = usize::try_from(spec.max_threads).map_err(|error| { + CommandFailure::new( + ErrorCode::SessionLimitExceeded, + format!("CPU max_threads exceed this platform: {error}"), + true, + ) + })?; + let stack_depth = u16::try_from(spec.stack_depth).map_err(|error| { + CommandFailure::new( + ErrorCode::SessionLimitExceeded, + format!("CPU stack_depth exceeds the perf ABI: {error}"), + true, + ) + })?; + let thread_ids = cpu_sampling::target_thread_ids(&spec.target).map_err(|error| { + CommandFailure::new(error.code(), error.to_string(), error.recoverable()) + })?; + Ok(CpuSamplingRequest { + thread_ids, + frequency_hz: spec.frequency_hz, + duration: Duration::from_millis(spec.duration_ms), + timeout: Duration::from_millis(spec.timeout_ms), + max_samples, + max_threads, + stack_depth, + }) +} + +fn cpu_collector_failure(error: &cpu_collector::CpuSamplingError) -> CommandFailure { + let code = match error { + cpu_collector::CpuSamplingError::InvalidRequest(_) + | cpu_collector::CpuSamplingError::Timeout => ErrorCode::SessionLimitExceeded, + cpu_collector::CpuSamplingError::Open { source, .. } + | cpu_collector::CpuSamplingError::Arm { source, .. } + if source.kind() == std::io::ErrorKind::PermissionDenied => + { + ErrorCode::PermissionDenied + } + cpu_collector::CpuSamplingError::Cleanup { .. } => ErrorCode::CleanupFailed, + cpu_collector::CpuSamplingError::NoThreadsAttached { .. } => ErrorCode::TargetExited, + cpu_collector::CpuSamplingError::Open { .. } + | cpu_collector::CpuSamplingError::Ring { .. } + | cpu_collector::CpuSamplingError::Arm { .. } => ErrorCode::Internal, + }; + let recoverable = !matches!(code, ErrorCode::Internal | ErrorCode::CleanupFailed); + CommandFailure::new(code, error.to_string(), recoverable) +} + +fn emit_cpu_inventory(result: &CpuSampleInventoryResult, json: bool) -> ExitCode { + if json { + println!( + "{}", + serde_json::to_string_pretty(result).expect("CPU inventory must serialize") + ); + } else { + println!( + "CPU sample inventory: {} samples", + result.collection.observed_samples + ); + println!( + "Stacks: {} groups, {} lost, {} unresolved frames", + result.collection.groups, + result.collection.lost_samples, + result.symbolization.unresolved_frames + ); + for hotspot in result.inventory.hotspots.iter().take(10) { + let label = hotspot + .frame + .symbol + .as_deref() + .or(hotspot.frame.module_path.as_deref()) + .map_or_else(|| format!("{:#x}", hotspot.frame.address), str::to_owned); + println!( + " {:.1}% inclusive, {:.1}% self {label}", + hotspot.inclusive_proportion * 100.0, + hotspot.exclusive_proportion * 100.0 + ); + } + for warning in &result.warnings { + println!("Warning: {}: {}", warning.code, warning.message); + } + } + ExitCode::SUCCESS +} + fn validate_measure_source(pid: Option, input: &[PathBuf]) -> Result<(), CommandFailure> { if pid.is_some() && !input.is_empty() { return Err(CommandFailure::new( @@ -943,6 +1246,26 @@ fn run_measure_spec( ); } }; + let value: serde_json::Value = match serde_json::from_slice(&bytes) { + Ok(value) => value, + Err(error) => { + return emit_error( + ErrorCode::InvalidEventSelector, + format!("invalid measurement spec {}: {error}", path.display()), + true, + json, + ); + } + }; + if value.get("sample_event").is_some() { + return run_cpu_sampling_file_spec( + value, + path, + cupti_socket.is_some() || agent_path.is_some(), + events_out.is_some() || format.is_some(), + json, + ); + } let spec: MeasurementSpec = match serde_json::from_slice(&bytes) { Ok(spec) => spec, Err(error) => { @@ -1011,6 +1334,35 @@ fn run_measure_spec( } } +fn run_cpu_sampling_file_spec( + value: serde_json::Value, + path: &Path, + has_device_options: bool, + has_artifact_options: bool, + json: bool, +) -> ExitCode { + if has_device_options || has_artifact_options { + return emit_error( + ErrorCode::InvalidEventSelector, + "CpuSamplingSpec does not accept CUPTI or event artifact options".to_owned(), + true, + json, + ); + } + let spec: CpuSamplingSpec = match serde_json::from_value(value) { + Ok(spec) => spec, + Err(error) => { + return emit_error( + ErrorCode::InvalidEventSelector, + format!("invalid CpuSamplingSpec {}: {error}", path.display()), + true, + json, + ); + } + }; + run_cpu_sampling_spec(&spec, None, json) +} + fn measurement_spec_capacity(spec: &MeasurementSpec) -> Result { let capacity = match spec.measurement_mode { MeasurementMode::Exact if spec.max_groups.is_some() => { diff --git a/xprobe/cli/tests/cpu_sampling.rs b/xprobe/cli/tests/cpu_sampling.rs new file mode 100644 index 0000000..0140b03 --- /dev/null +++ b/xprobe/cli/tests/cpu_sampling.rs @@ -0,0 +1,112 @@ +use std::{ + process::{Command, Stdio}, + thread, + time::Duration, +}; + +use xprobe_protocol::{CpuFrameLanguage, CpuSampleInventoryResult}; + +#[test] +#[ignore = "requires permission to open perf events for a sibling process"] +fn samples_and_symbolizes_a_busy_native_process() { + let mut target = Command::new("sh") + .args(["-c", "while :; do :; done"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("busy target must start"); + let output = Command::new(env!("CARGO_BIN_EXE_xprobe")) + .args([ + "measure", + "--pid", + &target.id().to_string(), + "--cpu-sample", + "--duration-ms", + "250", + "--frequency-hz", + "199", + "--json", + "--non-interactive", + "--no-color", + ]) + .output() + .expect("xprobe CPU sampling must run"); + target.kill().expect("busy target must stop"); + target.wait().expect("busy target must be reaped"); + + assert!( + output.status.success(), + "stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let result: CpuSampleInventoryResult = + serde_json::from_slice(&output.stdout).expect("stdout must contain CPU inventory JSON"); + assert!(result.collection.observed_samples > 0); + assert!(!result.inventory.stack_groups.is_empty()); + assert!( + result.inventory.hotspots.iter().any(|hotspot| { + hotspot.frame.language == CpuFrameLanguage::Native && hotspot.frame.symbol.is_some() + }), + "{}", + String::from_utf8_lossy(&output.stdout), + ); +} + +#[test] +#[ignore = "requires CPython perf trampoline support and perf event access"] +fn resolves_cpython_perf_map_frames() { + let mut target = Command::new("/usr/bin/python3") + .args([ + "-X", + "perf", + "-c", + "def leaf():\n return sum(range(100))\nwhile True:\n leaf()", + ]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("Python target must start"); + thread::sleep(Duration::from_millis(100)); + let output = Command::new(env!("CARGO_BIN_EXE_xprobe")) + .args([ + "measure", + "--pid", + &target.id().to_string(), + "--cpu-sample", + "--duration-ms", + "250", + "--frequency-hz", + "199", + "--json", + "--non-interactive", + "--no-color", + ]) + .output() + .expect("xprobe CPU sampling must run"); + target.kill().expect("Python target must stop"); + target.wait().expect("Python target must be reaped"); + + assert!( + output.status.success(), + "stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let result: CpuSampleInventoryResult = + serde_json::from_slice(&output.stdout).expect("stdout must contain CPU inventory JSON"); + assert!( + result.inventory.hotspots.iter().any(|hotspot| { + hotspot.frame.language == CpuFrameLanguage::Python + && hotspot + .frame + .symbol + .as_deref() + .is_some_and(|name| name.contains("leaf")) + }), + "{}", + String::from_utf8_lossy(&output.stdout) + ); +} diff --git a/xprobe/collector/Cargo.toml b/xprobe/collector/Cargo.toml index 2875a5c..adcf9b1 100644 --- a/xprobe/collector/Cargo.toml +++ b/xprobe/collector/Cargo.toml @@ -8,6 +8,7 @@ rust-version.workspace = true [dependencies] libbpf-rs.workspace = true +perf-event-open.workspace = true serde_json.workspace = true xprobe-protocol.workspace = true diff --git a/xprobe/collector/src/cpu_sampling.rs b/xprobe/collector/src/cpu_sampling.rs new file mode 100644 index 0000000..d3fa6c0 --- /dev/null +++ b/xprobe/collector/src/cpu_sampling.rs @@ -0,0 +1,421 @@ +use std::{ + error::Error, + fmt, io, thread, + time::{Duration, Instant}, +}; + +use perf_event_open::{ + config::{CallChain as CallChainConfig, Cpu, Opts, Priv as ExcludedPriv, Proc, SampleOn}, + count::Counter, + event::sw::Software, + sample::{ + Sampler, + record::{Record, sample::CallChain}, + }, +}; +const RING_PAGE_EXPONENT: u8 = 3; +const DRAIN_INTERVAL: Duration = Duration::from_millis(10); +const ESRCH: i32 = 3; + +#[derive(Debug, Clone)] +pub struct CpuSamplingRequest { + pub thread_ids: Vec, + pub frequency_hz: u64, + pub duration: Duration, + pub timeout: Duration, + pub max_samples: usize, + pub max_threads: usize, + pub stack_depth: u16, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RawCpuStack { + pub thread_id: u32, + pub addresses: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CpuSampleCapture { + pub stacks: Vec, + pub observed_samples: u64, + pub lost_samples: u64, + pub truncated_stacks: u64, + pub throttled_records: u64, + pub threads_observed: usize, + pub threads_attached: usize, + pub skipped_threads: Vec, + pub capacity_reached: bool, + pub elapsed: Duration, +} + +#[derive(Debug)] +pub enum CpuSamplingError { + InvalidRequest(&'static str), + Open { thread_id: u32, source: io::Error }, + Ring { thread_id: u32, source: io::Error }, + Arm { thread_id: u32, source: io::Error }, + Cleanup { failures: Vec }, + Timeout, + NoThreadsAttached { skipped_threads: Vec }, +} + +impl fmt::Display for CpuSamplingError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidRequest(message) => { + write!(formatter, "invalid CPU sampling request: {message}") + } + Self::Open { thread_id, source } => { + write!( + formatter, + "failed to open perf event for thread {thread_id}: {source}" + ) + } + Self::Ring { thread_id, source } => { + write!( + formatter, + "failed to map perf ring for thread {thread_id}: {source}" + ) + } + Self::Arm { thread_id, source } => { + write!( + formatter, + "failed to enable perf event for thread {thread_id}: {source}" + ) + } + Self::Cleanup { failures } => { + write!( + formatter, + "failed to disable perf events: {}", + failures.join("; ") + ) + } + Self::Timeout => formatter.write_str("CPU sampling timed out"), + Self::NoThreadsAttached { skipped_threads } => write!( + formatter, + "no target threads remained attachable; exited threads: {skipped_threads:?}" + ), + } + } +} + +impl Error for CpuSamplingError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Open { source, .. } | Self::Ring { source, .. } | Self::Arm { source, .. } => { + Some(source) + } + Self::InvalidRequest(_) + | Self::Cleanup { .. } + | Self::Timeout + | Self::NoThreadsAttached { .. } => None, + } + } +} + +struct ThreadSampler { + thread_id: u32, + counter: Counter, + sampler: Sampler, +} + +/// Collect bounded user-space CPU call chains from a fixed target thread snapshot. +/// +/// # Errors +/// +/// Returns [`CpuSamplingError`] for invalid bounds, perf setup failures, timeout, +/// or deterministic cleanup failures. +pub fn collect(request: &CpuSamplingRequest) -> Result { + validate_request(request)?; + let started = Instant::now(); + let deadline = started + .checked_add(request.timeout) + .ok_or(CpuSamplingError::InvalidRequest( + "timeout overflows the monotonic clock", + ))?; + let observed = request.thread_ids.len(); + let selected = request.thread_ids.iter().copied().take(request.max_threads); + let mut skipped_threads = request + .thread_ids + .iter() + .copied() + .skip(request.max_threads) + .collect::>(); + let opts = sampling_options(request); + let mut samplers = Vec::new(); + + for thread_id in selected { + if Instant::now() >= deadline { + return Err(CpuSamplingError::Timeout); + } + let counter = match Counter::new(Software::CpuClock, (Proc(thread_id), Cpu::ALL), &opts) { + Ok(counter) => counter, + Err(source) if source.raw_os_error() == Some(ESRCH) => { + skipped_threads.push(thread_id); + continue; + } + Err(source) => return Err(CpuSamplingError::Open { thread_id, source }), + }; + let sampler = counter + .sampler(RING_PAGE_EXPONENT) + .map_err(|source| CpuSamplingError::Ring { thread_id, source })?; + samplers.push(ThreadSampler { + thread_id, + counter, + sampler, + }); + } + if samplers.is_empty() { + return Err(CpuSamplingError::NoThreadsAttached { skipped_threads }); + } + + arm_all(&samplers)?; + let capture_started = Instant::now(); + let capture_deadline = + capture_started + .checked_add(request.duration) + .ok_or(CpuSamplingError::InvalidRequest( + "duration overflows the monotonic clock", + ))?; + let mut capture = CpuSampleCapture { + stacks: Vec::with_capacity(request.max_samples.min(4096)), + observed_samples: 0, + lost_samples: 0, + truncated_stacks: 0, + throttled_records: 0, + threads_observed: observed, + threads_attached: samplers.len(), + skipped_threads, + capacity_reached: false, + elapsed: Duration::ZERO, + }; + + while Instant::now() < capture_deadline && capture.observed_samples < request.max_samples as u64 + { + if Instant::now() >= deadline { + disable_all(&samplers)?; + return Err(CpuSamplingError::Timeout); + } + drain(&samplers, request, &mut capture); + let now = Instant::now(); + if now < capture_deadline && capture.observed_samples < request.max_samples as u64 { + thread::sleep(DRAIN_INTERVAL.min(capture_deadline.duration_since(now))); + } + } + + disable_all(&samplers)?; + drain(&samplers, request, &mut capture); + capture.capacity_reached = capture.observed_samples >= request.max_samples as u64; + capture.elapsed = capture_started.elapsed(); + Ok(capture) +} + +fn validate_request(request: &CpuSamplingRequest) -> Result<(), CpuSamplingError> { + if request.thread_ids.is_empty() { + return Err(CpuSamplingError::InvalidRequest( + "thread_ids must not be empty", + )); + } + if request.frequency_hz == 0 { + return Err(CpuSamplingError::InvalidRequest( + "frequency_hz must be positive", + )); + } + if request.duration.is_zero() { + return Err(CpuSamplingError::InvalidRequest( + "duration must be positive", + )); + } + if request.timeout < request.duration { + return Err(CpuSamplingError::InvalidRequest( + "timeout must be at least the capture duration", + )); + } + if request.max_samples == 0 { + return Err(CpuSamplingError::InvalidRequest( + "max_samples must be positive", + )); + } + if request.max_threads == 0 { + return Err(CpuSamplingError::InvalidRequest( + "max_threads must be positive", + )); + } + if request.stack_depth == 0 { + return Err(CpuSamplingError::InvalidRequest( + "stack_depth must be positive", + )); + } + Ok(()) +} + +fn sampling_options(request: &CpuSamplingRequest) -> Opts { + let mut opts = Opts { + sample_on: SampleOn::Freq(request.frequency_hz), + exclude: ExcludedPriv { + kernel: true, + hv: true, + host: false, + guest: true, + idle: true, + user: false, + }, + ..Opts::default() + }; + opts.sample_format.code_addr = true; + opts.sample_format.call_chain = Some(CallChainConfig { + exclude_user: false, + exclude_kernel: true, + defer_user: false, + max_stack_frames: request.stack_depth, + }); + opts +} + +fn arm_all(samplers: &[ThreadSampler]) -> Result<(), CpuSamplingError> { + for (index, sampler) in samplers.iter().enumerate() { + if let Err(source) = sampler.counter.enable() { + let mut cleanup = disable_subset(&samplers[..index]); + if cleanup.is_empty() { + return Err(CpuSamplingError::Arm { + thread_id: sampler.thread_id, + source, + }); + } + cleanup.push(format!( + "original enable failure for thread {}: {source}", + sampler.thread_id + )); + return Err(CpuSamplingError::Cleanup { failures: cleanup }); + } + } + Ok(()) +} + +fn disable_all(samplers: &[ThreadSampler]) -> Result<(), CpuSamplingError> { + let failures = disable_subset(samplers); + if failures.is_empty() { + Ok(()) + } else { + Err(CpuSamplingError::Cleanup { failures }) + } +} + +fn disable_subset(samplers: &[ThreadSampler]) -> Vec { + samplers + .iter() + .filter_map(|sampler| { + sampler + .counter + .disable() + .err() + .map(|error| format!("thread {}: {error}", sampler.thread_id)) + }) + .collect() +} + +fn drain(samplers: &[ThreadSampler], request: &CpuSamplingRequest, capture: &mut CpuSampleCapture) { + for sampler in samplers { + for (_, record) in sampler.sampler.iter() { + match record { + Record::Sample(sample) if capture.observed_samples < request.max_samples as u64 => { + capture.observed_samples += 1; + if let Some((addresses, truncated)) = + user_addresses(&sample, request.stack_depth) + { + capture.truncated_stacks += u64::from(truncated); + capture.stacks.push(RawCpuStack { + thread_id: sampler.thread_id, + addresses, + }); + } + } + Record::Sample(_) => { + capture.capacity_reached = true; + return; + } + Record::LostRecords(lost) => { + capture.lost_samples = capture.lost_samples.saturating_add(lost.lost_records); + } + Record::LostSamples(lost) => { + capture.lost_samples = capture.lost_samples.saturating_add(lost.lost_samples); + } + Record::Throttle(_) => { + capture.throttled_records = capture.throttled_records.saturating_add(1); + } + _ => {} + } + if capture.observed_samples >= request.max_samples as u64 { + capture.capacity_reached = true; + return; + } + } + } +} + +fn user_addresses( + sample: &perf_event_open::sample::record::sample::Sample, + stack_depth: u16, +) -> Option<(Vec, bool)> { + let mut addresses = sample + .call_chain + .as_ref() + .and_then(|chains| { + chains.iter().find_map(|chain| match chain { + CallChain::User(addresses) => Some(addresses.clone()), + _ => None, + }) + }) + .unwrap_or_default(); + if addresses.is_empty() { + addresses.extend(sample.code_addr.map(|(address, _)| address)); + } + addresses.retain(|address| *address != 0); + let truncated = addresses.len() >= usize::from(stack_depth); + addresses.truncate(usize::from(stack_depth)); + (!addresses.is_empty()).then_some((addresses, truncated)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn request() -> CpuSamplingRequest { + CpuSamplingRequest { + thread_ids: vec![1], + frequency_hz: 99, + duration: Duration::from_millis(10), + timeout: Duration::from_millis(20), + max_samples: 100, + max_threads: 8, + stack_depth: 64, + } + } + + #[test] + fn rejects_zero_resource_bounds() { + let mut value = request(); + value.max_samples = 0; + assert!(matches!( + validate_request(&value), + Err(CpuSamplingError::InvalidRequest(_)) + )); + + value = request(); + value.stack_depth = 0; + assert!(matches!( + validate_request(&value), + Err(CpuSamplingError::InvalidRequest(_)) + )); + } + + #[test] + fn rejects_a_timeout_shorter_than_capture() { + let mut value = request(); + value.timeout = Duration::from_millis(9); + assert!(matches!( + validate_request(&value), + Err(CpuSamplingError::InvalidRequest(_)) + )); + } +} diff --git a/xprobe/collector/src/lib.rs b/xprobe/collector/src/lib.rs index 7e354d2..6f4a23f 100644 --- a/xprobe/collector/src/lib.rs +++ b/xprobe/collector/src/lib.rs @@ -1,6 +1,7 @@ //! Host and device event collector interfaces. pub mod completed; +pub mod cpu_sampling; pub mod cupti; pub mod linux; pub mod uprobe; diff --git a/xprobe/core/src/cpu_sampling_analysis.rs b/xprobe/core/src/cpu_sampling_analysis.rs new file mode 100644 index 0000000..f79a915 --- /dev/null +++ b/xprobe/core/src/cpu_sampling_analysis.rs @@ -0,0 +1,807 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + error::Error, + fmt, fs, io, + path::{Path, PathBuf}, +}; + +use cpp_demangle::{DemangleOptions, Symbol as CppSymbol}; +use object::{Object, ObjectSegment, ObjectSymbol, SymbolKind}; +use xprobe_protocol::{ + CpuCaptureCompleteness, CpuFrameLanguage, CpuHotspot, CpuSampleCollectionSummary, + CpuSampleEvent, CpuSampleInventory, CpuSampleInventoryResult, CpuSamplingSpec, CpuStackFrame, + CpuStackGroup, CpuSymbolizationSummary, ErrorCode, ProcessReport, PythonSymbolizationStatus, + SchemaVersion, SessionStatus, Warning, +}; + +use crate::inspect::{self, InspectError}; + +const MAX_PERF_MAP_BYTES: u64 = 16 * 1024 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RawCpuSample { + pub thread_id: u32, + pub addresses: Vec, +} + +#[derive(Debug, Clone)] +pub struct CpuCaptureData { + pub samples: Vec, + pub observed_samples: u64, + pub lost_samples: u64, + pub truncated_stacks: u64, + pub throttled_records: u64, + pub threads_observed: usize, + pub threads_attached: usize, + pub skipped_threads: Vec, + pub capacity_reached: bool, + pub python_status: PythonSymbolizationStatus, +} + +#[derive(Debug)] +pub enum CpuAnalysisError { + Inspect(InspectError), + Io { path: PathBuf, source: io::Error }, + InvalidMaps { path: PathBuf, reason: String }, + InvalidPerfMap { path: PathBuf, reason: String }, + InvalidUtf8 { path: PathBuf }, + Limit { name: &'static str, value: u64 }, +} + +impl CpuAnalysisError { + #[must_use] + pub fn code(&self) -> ErrorCode { + match self { + Self::Inspect(error) => error.code(), + Self::Io { source, .. } if source.kind() == io::ErrorKind::PermissionDenied => { + ErrorCode::PermissionDenied + } + Self::Limit { .. } => ErrorCode::SessionLimitExceeded, + Self::Io { .. } + | Self::InvalidMaps { .. } + | Self::InvalidPerfMap { .. } + | Self::InvalidUtf8 { .. } => ErrorCode::Internal, + } + } + + #[must_use] + pub fn recoverable(&self) -> bool { + match self { + Self::Inspect(error) => error.recoverable(), + Self::Io { source, .. } => matches!( + source.kind(), + io::ErrorKind::NotFound | io::ErrorKind::PermissionDenied + ), + Self::Limit { .. } => true, + Self::InvalidMaps { .. } | Self::InvalidPerfMap { .. } | Self::InvalidUtf8 { .. } => { + false + } + } + } +} + +impl fmt::Display for CpuAnalysisError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Inspect(error) => error.fmt(formatter), + Self::Io { path, source } => { + write!(formatter, "failed to read {}: {source}", path.display()) + } + Self::InvalidMaps { path, reason } => { + write!(formatter, "invalid {}: {reason}", path.display()) + } + Self::InvalidPerfMap { path, reason } => { + write!( + formatter, + "invalid Python perf map {}: {reason}", + path.display() + ) + } + Self::InvalidUtf8 { path } => { + write!(formatter, "{} is not valid UTF-8", path.display()) + } + Self::Limit { name, value } => { + write!( + formatter, + "CPU analysis {name} exceeds the supported bound: {value}" + ) + } + } + } +} + +impl Error for CpuAnalysisError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Inspect(error) => Some(error), + Self::Io { source, .. } => Some(source), + Self::InvalidMaps { .. } + | Self::InvalidPerfMap { .. } + | Self::InvalidUtf8 { .. } + | Self::Limit { .. } => None, + } + } +} + +impl From for CpuAnalysisError { + fn from(error: InspectError) -> Self { + Self::Inspect(error) + } +} + +#[derive(Debug, Clone)] +struct MapRegion { + start: u64, + end: u64, + file_offset: u64, + path: PathBuf, +} + +#[derive(Debug, Clone)] +struct ModuleSymbol { + file_offset: u64, + size: u64, + name: String, +} + +#[derive(Debug, Clone)] +struct Module { + build_id: Option, + symbols: Vec, +} + +#[derive(Debug, Clone)] +struct PerfMapSymbol { + start: u64, + end: u64, + name: String, +} + +struct Symbolizer { + maps: Vec, + modules: BTreeMap, + python_symbols: Vec, + python_status: PythonSymbolizationStatus, + warnings: Vec, +} + +/// Aggregate and symbolize one completed bounded CPU capture. +/// +/// # Errors +/// +/// Returns [`CpuAnalysisError`] when target identity, procfs mappings, Python +/// perf-map data, or public integer bounds are invalid. +pub fn analyze( + report: &ProcessReport, + spec: &CpuSamplingSpec, + session_id: String, + capture: &CpuCaptureData, +) -> Result { + inspect::verify_target(&report.target)?; + let mut symbolizer = Symbolizer::load(report, capture.python_status)?; + let (raw_groups, group_capacity_reached) = + aggregate_raw_stacks(&capture.samples, spec.max_groups); + let grouped_samples = raw_groups.iter().map(|(_, count)| count).sum::(); + let sample_denominator = capture.observed_samples.max(1); + let stack_groups = raw_groups + .iter() + .map(|(addresses, samples)| CpuStackGroup { + frames: addresses + .iter() + .map(|address| symbolizer.symbolize(*address)) + .collect(), + samples: *samples, + proportion: ratio(*samples, sample_denominator), + }) + .collect::>(); + let hotspots = aggregate_hotspots( + &raw_groups, + &symbolizer, + sample_denominator, + spec.max_groups, + ); + let symbolization = summarize_symbolization(&stack_groups, symbolizer.python_status); + let mut warnings = std::mem::take(&mut symbolizer.warnings); + append_capture_warnings(capture, &mut warnings); + if group_capacity_reached { + warnings.push(warning( + "CPU_GROUP_CAPACITY_REACHED", + "distinct stack groups exceeded max_groups; only the heaviest groups and hotspots were retained", + )); + } + if symbolization.python_status == PythonSymbolizationStatus::Active + && symbolization.resolved_python_frames == 0 + { + warnings.push(warning( + "PYTHON_FRAMES_UNRESOLVED", + "the CPython perf map is active, but sampled callchains contained no resolvable Python trampoline frames", + )); + } + inspect::verify_target(&report.target)?; + + Ok(CpuSampleInventoryResult { + schema_version: SchemaVersion::current(), + ok: true, + session_id, + status: SessionStatus::Completed, + target: report.target.clone(), + inventory: CpuSampleInventory { + name: spec.name.clone(), + sample_event: CpuSampleEvent::CpuClock, + frequency_hz: spec.frequency_hz, + duration_ms: spec.duration_ms, + stack_groups, + hotspots, + }, + collection: CpuSampleCollectionSummary { + completeness: if capture_is_complete(capture, grouped_samples, group_capacity_reached) { + CpuCaptureCompleteness::Complete + } else { + CpuCaptureCompleteness::Incomplete + }, + observed_samples: capture.observed_samples, + grouped_samples, + lost_samples: capture.lost_samples, + sample_capacity: spec.max_samples, + group_capacity: spec.max_groups, + groups: u64::try_from(raw_groups.len()).map_err(|_| CpuAnalysisError::Limit { + name: "group count", + value: u64::MAX, + })?, + table_utilization: ratio( + u64::try_from(raw_groups.len()).unwrap_or(u64::MAX), + spec.max_groups.max(1), + ), + stack_depth: spec.stack_depth, + truncated_stacks: capture.truncated_stacks, + threads_observed: u32::try_from(capture.threads_observed).map_err(|_| { + CpuAnalysisError::Limit { + name: "observed thread count", + value: u64::MAX, + } + })?, + threads_attached: u32::try_from(capture.threads_attached).map_err(|_| { + CpuAnalysisError::Limit { + name: "attached thread count", + value: u64::MAX, + } + })?, + thread_capacity: spec.max_threads, + }, + symbolization, + warnings, + }) +} + +impl Symbolizer { + fn load( + report: &ProcessReport, + python_status: PythonSymbolizationStatus, + ) -> Result { + let maps_path = PathBuf::from(format!("/proc/{}/maps", report.target.pid)); + let maps_text = fs::read_to_string(&maps_path).map_err(|source| CpuAnalysisError::Io { + path: maps_path.clone(), + source, + })?; + let maps = parse_executable_maps(&maps_text, &maps_path)?; + let mut modules = BTreeMap::new(); + let mut warnings = Vec::new(); + for path in maps + .iter() + .map(|region| ®ion.path) + .collect::>() + { + match load_module(path) { + Ok(module) => { + modules.insert(path.clone(), module); + } + Err(message) => warnings.push(warning( + "NATIVE_MODULE_UNAVAILABLE", + &format!("could not symbolize {}: {message}", path.display()), + )), + } + } + let (python_symbols, python_status) = load_python_symbols(report, python_status)?; + Ok(Self { + maps, + modules, + python_symbols, + python_status, + warnings, + }) + } + + fn symbolize(&self, address: u64) -> CpuStackFrame { + if let Some(symbol) = find_perf_symbol(&self.python_symbols, address) { + return CpuStackFrame { + address, + module_path: None, + build_id: None, + file_offset: None, + symbol: Some(symbol.name.clone()), + symbol_offset: Some(address - symbol.start), + language: CpuFrameLanguage::Python, + source_path: None, + line: None, + }; + } + let Some(mapping) = self + .maps + .iter() + .find(|mapping| address >= mapping.start && address < mapping.end) + else { + return unresolved_frame(address); + }; + let file_offset = mapping.file_offset + (address - mapping.start); + let module = self.modules.get(&mapping.path); + let symbol = module.and_then(|module| find_module_symbol(&module.symbols, file_offset)); + CpuStackFrame { + address, + module_path: Some(mapping.path.to_string_lossy().into_owned()), + build_id: module.and_then(|module| module.build_id.clone()), + file_offset: Some(file_offset), + symbol: symbol.map(|symbol| symbol.name.clone()), + symbol_offset: symbol.map(|symbol| file_offset - symbol.file_offset), + language: CpuFrameLanguage::Native, + source_path: None, + line: None, + } + } +} + +fn aggregate_raw_stacks(samples: &[RawCpuSample], max_groups: u64) -> (Vec<(Vec, u64)>, bool) { + let mut groups = BTreeMap::, u64>::new(); + for sample in samples { + groups + .entry(sample.addresses.clone()) + .and_modify(|count| *count = count.saturating_add(1)) + .or_insert(1); + } + let mut groups = groups.into_iter().collect::>(); + groups.sort_by(|(left_stack, left_count), (right_stack, right_count)| { + right_count + .cmp(left_count) + .then_with(|| left_stack.cmp(right_stack)) + }); + let capacity = usize::try_from(max_groups).unwrap_or(usize::MAX); + let capacity_reached = groups.len() > capacity; + groups.truncate(capacity); + (groups, capacity_reached) +} + +fn aggregate_hotspots( + groups: &[(Vec, u64)], + symbolizer: &Symbolizer, + denominator: u64, + max_hotspots: u64, +) -> Vec { + let mut counts = BTreeMap::::new(); + for (stack, samples) in groups { + if let Some(address) = stack.first() { + let entry = counts.entry(*address).or_default(); + entry.1 = entry.1.saturating_add(*samples); + } + let mut seen = BTreeSet::new(); + for address in stack { + if seen.insert(*address) { + let entry = counts.entry(*address).or_default(); + entry.0 = entry.0.saturating_add(*samples); + } + } + } + let mut hotspots = counts + .into_iter() + .map(|(address, (inclusive, exclusive))| { + let frame = symbolizer.symbolize(address); + let (entry_selector_hint, return_selector_hint) = selector_hints(&frame); + CpuHotspot { + frame, + inclusive_samples: inclusive, + exclusive_samples: exclusive, + inclusive_proportion: ratio(inclusive, denominator), + exclusive_proportion: ratio(exclusive, denominator), + entry_selector_hint, + return_selector_hint, + } + }) + .collect::>(); + hotspots.sort_by(|left, right| { + right + .inclusive_samples + .cmp(&left.inclusive_samples) + .then_with(|| right.exclusive_samples.cmp(&left.exclusive_samples)) + .then_with(|| left.frame.address.cmp(&right.frame.address)) + }); + hotspots.truncate(usize::try_from(max_hotspots).unwrap_or(usize::MAX)); + hotspots +} + +fn selector_hints(frame: &CpuStackFrame) -> (Option, Option) { + if frame.language != CpuFrameLanguage::Native { + return (None, None); + } + let (Some(path), Some(offset)) = (&frame.module_path, frame.file_offset) else { + return (None, None); + }; + ( + Some(format!("uprobe:{path}:+0x{offset:x}:entry")), + Some(format!("uprobe:{path}:+0x{offset:x}:return")), + ) +} + +fn summarize_symbolization( + groups: &[CpuStackGroup], + python_status: PythonSymbolizationStatus, +) -> CpuSymbolizationSummary { + let mut summary = CpuSymbolizationSummary { + total_frames: 0, + resolved_native_frames: 0, + resolved_python_frames: 0, + unresolved_frames: 0, + python_status, + }; + for group in groups { + for frame in &group.frames { + summary.total_frames += 1; + match (frame.language, frame.symbol.is_some()) { + (CpuFrameLanguage::Native, true) => summary.resolved_native_frames += 1, + (CpuFrameLanguage::Python, true) => summary.resolved_python_frames += 1, + _ => summary.unresolved_frames += 1, + } + } + } + summary +} + +fn append_capture_warnings(capture: &CpuCaptureData, warnings: &mut Vec) { + if capture.capacity_reached { + warnings.push(warning( + "CPU_SAMPLE_CAPACITY_REACHED", + "the configured sample capacity was reached; shorten or lower the sampling frequency before interpreting proportions", + )); + } + if capture.lost_samples != 0 { + warnings.push(warning( + "CPU_SAMPLES_LOST", + &format!("perf reported {} lost samples", capture.lost_samples), + )); + } + if capture.throttled_records != 0 { + warnings.push(warning( + "CPU_SAMPLING_THROTTLED", + &format!( + "perf reported {} throttle records", + capture.throttled_records + ), + )); + } + if !capture.skipped_threads.is_empty() { + warnings.push(warning( + "CPU_THREADS_NOT_ATTACHED", + &format!( + "{} observed threads exited or exceeded max_threads before attachment", + capture.skipped_threads.len() + ), + )); + } +} + +fn capture_is_complete( + capture: &CpuCaptureData, + grouped_samples: u64, + group_capacity_reached: bool, +) -> bool { + !capture.capacity_reached + && !group_capacity_reached + && capture.lost_samples == 0 + && capture.throttled_records == 0 + && capture.skipped_threads.is_empty() + && grouped_samples == capture.observed_samples +} + +fn parse_executable_maps(text: &str, path: &Path) -> Result, CpuAnalysisError> { + text.lines() + .filter_map(|line| match parse_map_line(line, path) { + Ok(Some(mapping)) => Some(Ok(mapping)), + Ok(None) => None, + Err(error) => Some(Err(error)), + }) + .collect() +} + +fn parse_map_line(line: &str, path: &Path) -> Result, CpuAnalysisError> { + let mut fields = line.split_whitespace(); + let range = required_field(&mut fields, path, "address range")?; + let permissions = required_field(&mut fields, path, "permissions")?; + let offset = required_field(&mut fields, path, "file offset")?; + let _device = required_field(&mut fields, path, "device")?; + let _inode = required_field(&mut fields, path, "inode")?; + let mapped_path = fields.collect::>().join(" "); + if mapped_path.is_empty() { + return Ok(None); + } + if !permissions.contains('x') || !mapped_path.starts_with('/') { + return Ok(None); + } + let mapped_path = mapped_path + .strip_suffix(" (deleted)") + .unwrap_or(&mapped_path); + let (start, end) = range + .split_once('-') + .ok_or_else(|| invalid_maps(path, "invalid address range"))?; + let start = parse_hex(start, path, "invalid mapping start")?; + let end = parse_hex(end, path, "invalid mapping end")?; + let file_offset = parse_hex(offset, path, "invalid mapping file offset")?; + if start >= end { + return Err(invalid_maps(path, "mapping start must precede its end")); + } + Ok(Some(MapRegion { + start, + end, + file_offset, + path: PathBuf::from(mapped_path), + })) +} + +fn required_field<'a>( + fields: &mut impl Iterator, + path: &Path, + name: &str, +) -> Result<&'a str, CpuAnalysisError> { + fields + .next() + .ok_or_else(|| invalid_maps(path, &format!("missing {name}"))) +} + +fn invalid_maps(path: &Path, reason: &str) -> CpuAnalysisError { + CpuAnalysisError::InvalidMaps { + path: path.to_owned(), + reason: reason.to_owned(), + } +} + +fn parse_hex(value: &str, path: &Path, reason: &str) -> Result { + u64::from_str_radix(value, 16).map_err(|_| invalid_maps(path, reason)) +} + +fn load_module(path: &Path) -> Result { + let bytes = fs::read(path).map_err(|error| error.to_string())?; + let file = object::File::parse(bytes.as_slice()).map_err(|error| error.to_string())?; + let build_id = file + .build_id() + .map_err(|error| error.to_string())? + .map(hex_encode); + let mut symbols = BTreeMap::::new(); + for symbol in file.dynamic_symbols().chain(file.symbols()) { + if !symbol.is_definition() || symbol.kind() != SymbolKind::Text { + continue; + } + let Ok(name) = symbol.name() else { + continue; + }; + let Some(file_offset) = virtual_address_to_file_offset(&file, symbol.address()) else { + continue; + }; + let name = demangle_cpp(name).unwrap_or_else(|| name.to_owned()); + symbols + .entry(file_offset) + .and_modify(|current| current.size = current.size.max(symbol.size())) + .or_insert(ModuleSymbol { + file_offset, + size: symbol.size(), + name, + }); + } + Ok(Module { + build_id, + symbols: symbols.into_values().collect(), + }) +} + +fn virtual_address_to_file_offset(file: &object::File<'_>, address: u64) -> Option { + file.segments().find_map(|segment| { + let delta = address.checked_sub(segment.address())?; + let (file_offset, file_size) = segment.file_range(); + (delta < file_size).then(|| file_offset + delta) + }) +} + +fn find_module_symbol(symbols: &[ModuleSymbol], offset: u64) -> Option<&ModuleSymbol> { + let index = symbols.partition_point(|symbol| symbol.file_offset <= offset); + let symbol = symbols.get(index.checked_sub(1)?)?; + let next_start = symbols.get(index).map(|next| next.file_offset); + let within_size = symbol.size != 0 && offset - symbol.file_offset < symbol.size; + let before_next = symbol.size == 0 && next_start.is_none_or(|next| offset < next); + (within_size || before_next).then_some(symbol) +} + +fn load_python_symbols( + report: &ProcessReport, + detected_status: PythonSymbolizationStatus, +) -> Result<(Vec, PythonSymbolizationStatus), CpuAnalysisError> { + if matches!( + detected_status, + PythonSymbolizationStatus::NotDetected | PythonSymbolizationStatus::Unsupported + ) { + return Ok((Vec::new(), detected_status)); + } + let path = PathBuf::from(format!( + "/proc/{}/root/tmp/perf-{}.map", + report.target.pid, report.target.pid + )); + let metadata = match fs::metadata(&path) { + Ok(metadata) => metadata, + Err(source) if source.kind() == io::ErrorKind::NotFound => { + return Ok((Vec::new(), PythonSymbolizationStatus::Inactive)); + } + Err(source) => return Err(CpuAnalysisError::Io { path, source }), + }; + if metadata.len() > MAX_PERF_MAP_BYTES { + return Err(CpuAnalysisError::Limit { + name: "Python perf map bytes", + value: metadata.len(), + }); + } + let bytes = fs::read(&path).map_err(|source| CpuAnalysisError::Io { + path: path.clone(), + source, + })?; + let text = std::str::from_utf8(&bytes) + .map_err(|_| CpuAnalysisError::InvalidUtf8 { path: path.clone() })?; + let symbols = parse_perf_map(text, &path)?; + let status = if symbols.is_empty() { + PythonSymbolizationStatus::Inactive + } else { + PythonSymbolizationStatus::Active + }; + Ok((symbols, status)) +} + +fn parse_perf_map(text: &str, path: &Path) -> Result, CpuAnalysisError> { + let mut symbols = Vec::new(); + for (index, line) in text.lines().enumerate() { + if line.is_empty() { + continue; + } + let mut fields = line.splitn(3, ' '); + let start = fields.next().unwrap_or_default(); + let size = fields.next().unwrap_or_default(); + let name = fields.next().unwrap_or_default(); + let start = + u64::from_str_radix(start, 16).map_err(|_| CpuAnalysisError::InvalidPerfMap { + path: path.to_owned(), + reason: format!("line {} has an invalid address", index + 1), + })?; + let size = u64::from_str_radix(size, 16).map_err(|_| CpuAnalysisError::InvalidPerfMap { + path: path.to_owned(), + reason: format!("line {} has an invalid size", index + 1), + })?; + if size == 0 || name.is_empty() { + return Err(CpuAnalysisError::InvalidPerfMap { + path: path.to_owned(), + reason: format!("line {} has an empty symbol or zero size", index + 1), + }); + } + let end = start + .checked_add(size) + .ok_or_else(|| CpuAnalysisError::InvalidPerfMap { + path: path.to_owned(), + reason: format!("line {} address range overflows", index + 1), + })?; + symbols.push(PerfMapSymbol { + start, + end, + name: name.to_owned(), + }); + } + symbols.sort_by_key(|symbol| symbol.start); + Ok(symbols) +} + +fn find_perf_symbol(symbols: &[PerfMapSymbol], address: u64) -> Option<&PerfMapSymbol> { + let index = symbols.partition_point(|symbol| symbol.start <= address); + let symbol = symbols.get(index.checked_sub(1)?)?; + (address < symbol.end).then_some(symbol) +} + +fn unresolved_frame(address: u64) -> CpuStackFrame { + CpuStackFrame { + address, + module_path: None, + build_id: None, + file_offset: None, + symbol: None, + symbol_offset: None, + language: CpuFrameLanguage::Unknown, + source_path: None, + line: None, + } +} + +fn warning(code: &str, message: &str) -> Warning { + Warning { + code: code.to_owned(), + message: message.to_owned(), + details: BTreeMap::new(), + } +} + +fn demangle_cpp(name: &str) -> Option { + if !name.starts_with("_Z") { + return None; + } + CppSymbol::new(name) + .ok()? + .demangle(&DemangleOptions::default()) + .ok() +} + +fn hex_encode(bytes: &[u8]) -> String { + use fmt::Write as _; + + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + write!(output, "{byte:02x}").expect("writing to a string cannot fail"); + } + output +} + +#[allow(clippy::cast_precision_loss)] +fn ratio(numerator: u64, denominator: u64) -> f64 { + numerator as f64 / denominator as f64 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keeps_the_heaviest_stack_groups_deterministically() { + let samples = vec![ + raw(&[1, 2]), + raw(&[3, 4]), + raw(&[1, 2]), + raw(&[5, 6]), + raw(&[3, 4]), + raw(&[1, 2]), + ]; + assert_eq!( + aggregate_raw_stacks(&samples, 2), + (vec![(vec![1, 2], 3), (vec![3, 4], 2)], true) + ); + } + + #[test] + fn parses_executable_maps_with_spaces_and_deleted_suffixes() { + let mappings = parse_executable_maps( + "55550000-55551000 r-xp 00001000 08:01 42 /tmp/app name (deleted)\n\ + 7fff0000-7fff1000 rw-p 00000000 00:00 0 [stack]\n", + Path::new("maps"), + ) + .unwrap(); + assert_eq!(mappings.len(), 1); + assert_eq!(mappings[0].path, PathBuf::from("/tmp/app name")); + assert_eq!(mappings[0].file_offset, 0x1000); + } + + #[test] + fn parses_and_resolves_python_perf_map_symbols() { + let symbols = parse_perf_map( + "7f000000 20 py::worker (/srv/app.py:7)\n7f000020 10 py::leaf\n", + Path::new("perf.map"), + ) + .unwrap(); + assert_eq!( + find_perf_symbol(&symbols, 0x7f00_0005).unwrap().name, + "py::worker (/srv/app.py:7)" + ); + assert!(find_perf_symbol(&symbols, 0x7f00_0030).is_none()); + } + + fn raw(addresses: &[u64]) -> RawCpuSample { + RawCpuSample { + thread_id: 1, + addresses: addresses.to_vec(), + } + } +} diff --git a/xprobe/core/src/lib.rs b/xprobe/core/src/lib.rs index a9ec004..67dbebb 100644 --- a/xprobe/core/src/lib.rs +++ b/xprobe/core/src/lib.rs @@ -1,6 +1,7 @@ //! Measurement orchestration and domain logic for xprobe. pub mod cpu_sampling; +pub mod cpu_sampling_analysis; pub mod cupti_compat; pub mod discover; pub mod doctor; diff --git a/xprobe/protocol/src/cpu_sampling.rs b/xprobe/protocol/src/cpu_sampling.rs index a1538bd..aae2e2d 100644 --- a/xprobe/protocol/src/cpu_sampling.rs +++ b/xprobe/protocol/src/cpu_sampling.rs @@ -1,10 +1,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::{ - CaptureCompleteness, CheckResult, SchemaVersion, SessionStatus, TargetIdentity, - ValidationIssue, Warning, -}; +use crate::{CheckResult, SchemaVersion, SessionStatus, TargetIdentity, ValidationIssue, Warning}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] @@ -93,7 +90,7 @@ pub enum PythonSymbolizationStatus { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct CpuSampleCollectionSummary { - pub completeness: CaptureCompleteness, + pub completeness: CpuCaptureCompleteness, pub observed_samples: u64, pub grouped_samples: u64, pub lost_samples: u64, @@ -108,6 +105,13 @@ pub struct CpuSampleCollectionSummary { pub thread_capacity: u32, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CpuCaptureCompleteness { + Complete, + Incomplete, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct CpuSymbolizationSummary { diff --git a/xprobe/protocol/src/lib.rs b/xprobe/protocol/src/lib.rs index 0e74690..18ac4bf 100644 --- a/xprobe/protocol/src/lib.rs +++ b/xprobe/protocol/src/lib.rs @@ -19,10 +19,10 @@ pub use capability::{ }; pub use capture::HostCaptureResult; pub use cpu_sampling::{ - CpuFrameLanguage, CpuHotspot, CpuSampleCollectionSummary, CpuSampleEvent, CpuSampleInventory, - CpuSampleInventoryResult, CpuSamplingRequirements, CpuSamplingSpec, - CpuSamplingValidationResult, CpuStackFrame, CpuStackGroup, CpuSymbolizationSummary, - PythonSymbolizationStatus, + CpuCaptureCompleteness, CpuFrameLanguage, CpuHotspot, CpuSampleCollectionSummary, + CpuSampleEvent, CpuSampleInventory, CpuSampleInventoryResult, CpuSamplingRequirements, + CpuSamplingSpec, CpuSamplingValidationResult, CpuStackFrame, CpuStackGroup, + CpuSymbolizationSummary, PythonSymbolizationStatus, }; pub use discover::{CudaProcessCandidate, DiscoveryResult}; pub use error::{ErrorCode, ErrorResponse, XprobeError}; From 2b80f6ff1957b1a5750c5106049e223971e4ecd8 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 1 Aug 2026 04:45:43 +0800 Subject: [PATCH 04/15] =?UTF-8?q?=E2=9C=A8=20feat:=20aggregate=20syscall?= =?UTF-8?q?=20lifecycles=20in=20BPF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bpf/xprobe.bpf.c | 173 +++++++ schemas/syscall-aggregate-result.schema.json | 275 ++++++++++ schemas/syscall-aggregate-spec.schema.json | 76 +++ .../syscall-aggregate-validate.schema.json | 183 +++++++ tests/integration/run-linux-live.sh | 14 +- tests/integration/test_linux.py | 17 +- xprobe/cli/src/main.rs | 294 ++++++++++- xprobe/cli/tests/validate.rs | 25 +- xprobe/collector/src/lib.rs | 1 + xprobe/collector/src/syscall_aggregate.rs | 471 ++++++++++++++++++ xprobe/core/src/lib.rs | 1 + xprobe/core/src/syscall_aggregate.rs | 371 ++++++++++++++ xprobe/protocol/src/lib.rs | 6 + xprobe/protocol/src/schema.rs | 18 +- xprobe/protocol/src/syscall_aggregate.rs | 96 ++++ xprobe/protocol/tests/contracts.rs | 68 ++- 16 files changed, 2073 insertions(+), 16 deletions(-) create mode 100644 schemas/syscall-aggregate-result.schema.json create mode 100644 schemas/syscall-aggregate-spec.schema.json create mode 100644 schemas/syscall-aggregate-validate.schema.json create mode 100644 xprobe/collector/src/syscall_aggregate.rs create mode 100644 xprobe/core/src/syscall_aggregate.rs create mode 100644 xprobe/protocol/src/syscall_aggregate.rs diff --git a/bpf/xprobe.bpf.c b/bpf/xprobe.bpf.c index bafa1ec..ac152cf 100644 --- a/bpf/xprobe.bpf.c +++ b/bpf/xprobe.bpf.c @@ -6,6 +6,11 @@ #define __type(name, value) value *name static void *(*bpf_map_lookup_elem)(void *map, const void *key) = (void *)BPF_FUNC_map_lookup_elem; +static long (*bpf_map_update_elem)(void *map, const void *key, + const void *value, __u64 flags) = + (void *)BPF_FUNC_map_update_elem; +static long (*bpf_map_delete_elem)(void *map, const void *key) = + (void *)BPF_FUNC_map_delete_elem; static __u64 (*bpf_ktime_get_ns)(void) = (void *)BPF_FUNC_ktime_get_ns; static __u32 (*bpf_get_smp_processor_id)(void) = (void *)BPF_FUNC_get_smp_processor_id; static long (*bpf_probe_read_kernel)(void *dst, __u32 size, @@ -54,6 +59,34 @@ struct xprobe_linux_event { __u32 probe_id; }; +struct xprobe_syscall_aggregate_config { + __u64 pidns_dev; + __u64 pidns_ino; + __u32 target_pid; + __u32 armed; +}; + +struct xprobe_syscall_start { + __u64 timestamp_ns; + __u32 syscall_number; + __u32 reserved; +}; + +struct xprobe_syscall_aggregate { + __u64 count; + __u64 errors; + __u64 total_duration_ns; + __u64 min_duration_ns; + __u64 max_duration_ns; +}; + +struct xprobe_syscall_aggregate_summary { + __u64 entries; + __u64 exits; + __u64 unmatched_exits; + __u64 dropped_aggregates; +}; + struct xprobe_raw_tracepoint_context { __u64 arguments[2]; }; @@ -134,6 +167,34 @@ struct { __uint(max_entries, 256 * 1024); } linux_events SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct xprobe_syscall_aggregate_config); +} syscall_aggregate_config SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, 1024); + __type(key, __u32); + __type(value, struct xprobe_syscall_start); +} syscall_aggregate_inflight SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_HASH); + __uint(max_entries, 256); + __type(key, __u32); + __type(value, struct xprobe_syscall_aggregate); +} syscall_aggregate_groups SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct xprobe_syscall_aggregate_summary); +} syscall_aggregate_summary SEC(".maps"); + SEC("uprobe") int xprobe_handle_uprobe(void *context) { @@ -317,4 +378,116 @@ int xprobe_handle_syscall_exit(struct xprobe_raw_tracepoint_context *context) return xprobe_emit_linux_event(probe_id, values); } +static __attribute__((always_inline)) int +xprobe_target_syscall_aggregate(struct xprobe_syscall_aggregate_config *current, + struct bpf_pidns_info *nsdata) +{ + return current && current->armed && + bpf_get_ns_current_pid_tgid(current->pidns_dev, current->pidns_ino, + nsdata, sizeof(*nsdata)) == 0 && + current->target_pid == nsdata->tgid; +} + +static __attribute__((always_inline)) void +xprobe_count_syscall_summary(__u64 *field) +{ + if (field) + __sync_fetch_and_add(field, 1); +} + +SEC("raw_tracepoint") +int xprobe_aggregate_syscall_entry(struct xprobe_raw_tracepoint_context *context) +{ + __u32 zero = 0; + struct xprobe_syscall_aggregate_config *current = + bpf_map_lookup_elem(&syscall_aggregate_config, &zero); + struct xprobe_syscall_aggregate_summary *summary; + struct bpf_pidns_info nsdata = {}; + struct xprobe_syscall_start start; + __u32 thread_id; + + if (!xprobe_target_syscall_aggregate(current, &nsdata) || + (__s64)context->arguments[1] < 0) + return 0; + thread_id = nsdata.pid; + start.timestamp_ns = bpf_ktime_get_ns(); + start.syscall_number = (__u32)context->arguments[1]; + start.reserved = 0; + summary = bpf_map_lookup_elem(&syscall_aggregate_summary, &zero); + if (bpf_map_update_elem(&syscall_aggregate_inflight, &thread_id, &start, + BPF_ANY) != 0) { + if (summary) + xprobe_count_syscall_summary(&summary->dropped_aggregates); + return 0; + } + if (summary) + xprobe_count_syscall_summary(&summary->entries); + return 0; +} + +SEC("raw_tracepoint") +int xprobe_aggregate_syscall_exit(struct xprobe_raw_tracepoint_context *context) +{ + __u32 zero = 0; + struct xprobe_syscall_aggregate_config *current = + bpf_map_lookup_elem(&syscall_aggregate_config, &zero); + struct xprobe_syscall_aggregate_summary *summary; + struct xprobe_syscall_aggregate *aggregate; + struct xprobe_syscall_aggregate initial = {}; + struct xprobe_syscall_start *start; + struct bpf_pidns_info nsdata = {}; + __u64 duration_ns; + __u32 syscall_number; + __u32 thread_id; + + if (!xprobe_target_syscall_aggregate(current, &nsdata)) + return 0; + thread_id = nsdata.pid; + summary = bpf_map_lookup_elem(&syscall_aggregate_summary, &zero); + start = bpf_map_lookup_elem(&syscall_aggregate_inflight, &thread_id); + if (!start) { + if (summary) + xprobe_count_syscall_summary(&summary->unmatched_exits); + return 0; + } + syscall_number = start->syscall_number; + duration_ns = bpf_ktime_get_ns() - start->timestamp_ns; + bpf_map_delete_elem(&syscall_aggregate_inflight, &thread_id); + if (summary) + xprobe_count_syscall_summary(&summary->exits); + + aggregate = bpf_map_lookup_elem(&syscall_aggregate_groups, &syscall_number); + if (!aggregate) { + initial.min_duration_ns = duration_ns; + if (bpf_map_update_elem(&syscall_aggregate_groups, &syscall_number, + &initial, BPF_NOEXIST) != 0) { + aggregate = bpf_map_lookup_elem(&syscall_aggregate_groups, + &syscall_number); + if (!aggregate) { + if (summary) + xprobe_count_syscall_summary(&summary->dropped_aggregates); + return 0; + } + } else { + aggregate = bpf_map_lookup_elem(&syscall_aggregate_groups, + &syscall_number); + } + } + if (!aggregate) { + if (summary) + xprobe_count_syscall_summary(&summary->dropped_aggregates); + return 0; + } + aggregate->count += 1; + aggregate->total_duration_ns += duration_ns; + if ((__s64)context->arguments[1] < 0) + aggregate->errors += 1; + if (aggregate->min_duration_ns == 0 || + duration_ns < aggregate->min_duration_ns) + aggregate->min_duration_ns = duration_ns; + if (duration_ns > aggregate->max_duration_ns) + aggregate->max_duration_ns = duration_ns; + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/schemas/syscall-aggregate-result.schema.json b/schemas/syscall-aggregate-result.schema.json new file mode 100644 index 0000000..704f2d3 --- /dev/null +++ b/schemas/syscall-aggregate-result.schema.json @@ -0,0 +1,275 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "SyscallAggregateResult", + "type": "object", + "properties": { + "collection": { + "$ref": "#/$defs/SyscallAggregateCollectionSummary" + }, + "inventory": { + "$ref": "#/$defs/SyscallAggregateInventory" + }, + "ok": { + "type": "boolean" + }, + "schema_version": { + "$ref": "#/$defs/SchemaVersion" + }, + "session_id": { + "type": "string" + }, + "status": { + "$ref": "#/$defs/SessionStatus" + }, + "target": { + "$ref": "#/$defs/TargetIdentity" + }, + "warnings": { + "type": "array", + "default": [], + "items": { + "$ref": "#/$defs/Warning" + } + } + }, + "additionalProperties": false, + "required": [ + "schema_version", + "ok", + "session_id", + "status", + "target", + "inventory", + "collection" + ], + "$defs": { + "AggregateDuration": { + "type": "object", + "properties": { + "max": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "mean": { + "type": "number", + "format": "double" + }, + "min": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "total": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "min", + "mean", + "max", + "total" + ] + }, + "SchemaVersion": { + "type": "string", + "enum": [ + "2.0" + ] + }, + "SessionStatus": { + "type": "string", + "enum": [ + "completed", + "timed_out", + "cancelled", + "failed" + ] + }, + "SyscallAggregateCollectionSummary": { + "type": "object", + "properties": { + "completeness": { + "$ref": "#/$defs/SyscallAggregateCompleteness" + }, + "dropped_aggregates": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "group_capacity": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "groups": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "inflight_at_end": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "matched_exits": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "observed_entries": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "table_utilization": { + "type": "number", + "format": "double" + }, + "unmatched_exits": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "completeness", + "observed_entries", + "matched_exits", + "unmatched_exits", + "inflight_at_end", + "dropped_aggregates", + "group_capacity", + "groups", + "table_utilization" + ] + }, + "SyscallAggregateCompleteness": { + "type": "string", + "enum": [ + "complete", + "incomplete" + ] + }, + "SyscallAggregateGroup": { + "type": "object", + "properties": { + "count": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "duration_ns": { + "$ref": "#/$defs/AggregateDuration" + }, + "entry_selector_hint": { + "type": [ + "string", + "null" + ] + }, + "errors": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "exit_selector_hint": { + "type": [ + "string", + "null" + ] + }, + "syscall_name": { + "type": [ + "string", + "null" + ] + }, + "syscall_number": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "syscall_number", + "count", + "errors", + "duration_ns" + ] + }, + "SyscallAggregateInventory": { + "type": "object", + "properties": { + "duration_ms": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "groups": { + "type": "array", + "items": { + "$ref": "#/$defs/SyscallAggregateGroup" + } + }, + "name": { + "type": [ + "string", + "null" + ] + } + }, + "additionalProperties": false, + "required": [ + "duration_ms", + "groups" + ] + }, + "TargetIdentity": { + "type": "object", + "properties": { + "pid": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "process_start_time": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "pid", + "process_start_time" + ] + }, + "Warning": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "details": { + "type": "object", + "additionalProperties": true, + "default": {} + }, + "message": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "code", + "message" + ] + } + } +} diff --git a/schemas/syscall-aggregate-spec.schema.json b/schemas/syscall-aggregate-spec.schema.json new file mode 100644 index 0000000..f8d428c --- /dev/null +++ b/schemas/syscall-aggregate-spec.schema.json @@ -0,0 +1,76 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "SyscallAggregateSpec", + "type": "object", + "properties": { + "duration_ms": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "max_groups": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "max_inflight": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "schema_version": { + "$ref": "#/$defs/SchemaVersion" + }, + "target": { + "$ref": "#/$defs/TargetIdentity" + }, + "timeout_ms": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "schema_version", + "target", + "duration_ms", + "timeout_ms", + "max_groups", + "max_inflight" + ], + "$defs": { + "SchemaVersion": { + "type": "string", + "enum": [ + "2.0" + ] + }, + "TargetIdentity": { + "type": "object", + "properties": { + "pid": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "process_start_time": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "pid", + "process_start_time" + ] + } + } +} diff --git a/schemas/syscall-aggregate-validate.schema.json b/schemas/syscall-aggregate-validate.schema.json new file mode 100644 index 0000000..3f5beb5 --- /dev/null +++ b/schemas/syscall-aggregate-validate.schema.json @@ -0,0 +1,183 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "SyscallAggregateValidationResult", + "type": "object", + "properties": { + "ebpf": { + "$ref": "#/$defs/CheckResult" + }, + "issues": { + "type": "array", + "default": [], + "items": { + "$ref": "#/$defs/ValidationIssue" + } + }, + "ok": { + "type": "boolean" + }, + "requirements": { + "$ref": "#/$defs/SyscallAggregateRequirements" + }, + "schema_version": { + "$ref": "#/$defs/SchemaVersion" + }, + "target": { + "$ref": "#/$defs/TargetIdentity" + }, + "valid": { + "type": "boolean" + }, + "warnings": { + "type": "array", + "default": [], + "items": { + "$ref": "#/$defs/Warning" + } + } + }, + "additionalProperties": false, + "required": [ + "schema_version", + "ok", + "valid", + "target", + "requirements", + "ebpf" + ], + "$defs": { + "CheckResult": { + "type": "object", + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "status": { + "$ref": "#/$defs/CheckStatus" + } + }, + "additionalProperties": false, + "required": [ + "status" + ] + }, + "CheckStatus": { + "type": "string", + "enum": [ + "available", + "restricted", + "unavailable", + "unknown" + ] + }, + "ErrorCode": { + "type": "string", + "enum": [ + "PERMISSION_DENIED", + "TARGET_NOT_FOUND", + "TARGET_EXITED", + "TARGET_REUSED", + "AMBIGUOUS_TARGET", + "SYMBOL_NOT_FOUND", + "BINARY_NOT_MAPPED", + "INVALID_EVENT_SELECTOR", + "INVALID_CORRELATION_POLICY", + "CUPTI_NOT_AVAILABLE", + "CUPTI_AGENT_NOT_LOADED", + "CUDA_CONTEXT_NOT_FOUND", + "UNSUPPORTED_CUDA_VERSION", + "EVENT_RATE_TOO_HIGH", + "SESSION_LIMIT_EXCEEDED", + "NO_MATCHED_SAMPLES", + "HIGH_UNMATCHED_RATE", + "EVENTS_DROPPED", + "CLOCK_ALIGNMENT_FAILED", + "TRACE_EXPORT_FAILED", + "CLEANUP_FAILED", + "INTERNAL" + ] + }, + "SchemaVersion": { + "type": "string", + "enum": [ + "2.0" + ] + }, + "SyscallAggregateRequirements": { + "type": "object", + "properties": { + "needs_ebpf": { + "type": "boolean" + }, + "target_mutation": { + "type": "boolean" + } + }, + "additionalProperties": false, + "required": [ + "needs_ebpf", + "target_mutation" + ] + }, + "TargetIdentity": { + "type": "object", + "properties": { + "pid": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "process_start_time": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "pid", + "process_start_time" + ] + }, + "ValidationIssue": { + "type": "object", + "properties": { + "code": { + "$ref": "#/$defs/ErrorCode" + }, + "message": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "code", + "message" + ] + }, + "Warning": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "details": { + "type": "object", + "additionalProperties": true, + "default": {} + }, + "message": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "code", + "message" + ] + } + } +} diff --git a/tests/integration/run-linux-live.sh b/tests/integration/run-linux-live.sh index ed41e15..f645854 100755 --- a/tests/integration/run-linux-live.sh +++ b/tests/integration/run-linux-live.sh @@ -7,6 +7,7 @@ mmap_capture=/tmp/xprobe-mmap.json munmap_capture=/tmp/xprobe-munmap.json tracepoint_capture=/tmp/xprobe-tracepoint.json capacity_capture=/tmp/xprobe-capacity.json +syscall_inventory=/tmp/xprobe-syscall-inventory.json capacity_stderr=/tmp/xprobe-capacity.stderr "${target}" & @@ -15,10 +16,19 @@ cleanup() { kill "${target_pid}" 2>/dev/null || true wait "${target_pid}" 2>/dev/null || true rm -f "${mmap_capture}" "${munmap_capture}" "${tracepoint_capture}" \ - "${capacity_capture}" "${capacity_stderr}" + "${capacity_capture}" "${capacity_stderr}" "${syscall_inventory}" } trap cleanup EXIT +"${binary}" measure \ + --pid "${target_pid}" \ + --syscall-aggregate \ + --duration-ms 250 \ + --max-groups 64 \ + --max-inflight 64 \ + --timeout-ms 5000 \ + --json --non-interactive --no-color >"${syscall_inventory}" + "${binary}" measure \ --pid "${target_pid}" \ --from syscall:mmap:entry \ @@ -71,4 +81,6 @@ printf ',"tracepoint":' cat "${tracepoint_capture}" printf ',"capacity":' cat "${capacity_capture}" +printf ',"syscall_inventory":' +cat "${syscall_inventory}" printf '}\n' diff --git a/tests/integration/test_linux.py b/tests/integration/test_linux.py index 5437319..888315f 100755 --- a/tests/integration/test_linux.py +++ b/tests/integration/test_linux.py @@ -46,7 +46,8 @@ def main() -> None: assert_syscall_capture(captures["munmap"], "munmap") assert_tracepoint_capture(captures["tracepoint"]) assert_capacity_failure(captures["capacity"]) - print("captured mmap, munmap, and named tracepoint latency evidence") + assert_syscall_inventory(captures["syscall_inventory"]) + print("captured syscall inventory plus mmap, munmap, and named tracepoint latency evidence") def assert_syscall_capture(result: dict, name: str) -> None: @@ -100,5 +101,19 @@ def assert_capacity_failure(result: dict) -> None: assert result["error"]["details"]["record_capacity"] == 4 +def assert_syscall_inventory(result: dict) -> None: + assert result["ok"] is True + assert result["status"] == "completed" + assert result["collection"]["observed_entries"] > 0 + assert result["collection"]["matched_exits"] > 0 + assert result["collection"]["dropped_aggregates"] == 0 + groups = result["inventory"]["groups"] + assert groups + mmap = next(group for group in groups if group["syscall_name"] == "mmap") + assert mmap["count"] > 0 + assert mmap["duration_ns"]["total"] >= mmap["duration_ns"]["max"] + assert mmap["entry_selector_hint"] == "syscall:mmap:entry" + + if __name__ == "__main__": main() diff --git a/xprobe/cli/src/main.rs b/xprobe/cli/src/main.rs index d539d56..8304f29 100644 --- a/xprobe/cli/src/main.rs +++ b/xprobe/cli/src/main.rs @@ -22,12 +22,13 @@ use xprobe_collector::{ cpu_sampling::{self as cpu_collector, CpuSamplingRequest}, cupti, linux::{self, LinuxCaptureRequest}, + syscall_aggregate::{self as syscall_collector, SyscallAggregateRequest}, uprobe::{self, UprobeRequest}, }; use xprobe_core::{ cpu_sampling, cpu_sampling_analysis::{self, CpuCaptureData, RawCpuSample}, - cupti_compat, discover, doctor, inject, inspect, resolve, validate, + cupti_compat, discover, doctor, inject, inspect, resolve, syscall_aggregate, validate, }; use xprobe_correlator::{MeasureError, MeasureOptions, measure}; use xprobe_exporter::{events_to_chrome_trace, events_to_jsonl}; @@ -38,8 +39,9 @@ use xprobe_protocol::{ CpuSamplingValidationResult, CuptiCollectionSummary, DiscoveryResult, ErrorCode, ErrorResponse, Event, EventSource, EventType, ExportFormat, HostCaptureResult, MatchPolicy, MeasurementMode, MeasurementResult, MeasurementSpec, MemcpyKind, ProcessReport, ResolvedCudaSelector, - ResolvedLinuxSelector, ResolvedProbe, SchemaVersion, SessionStatus, TargetIdentity, - TraceExportResult, ValidationResult, Warning, XprobeError, + ResolvedLinuxSelector, ResolvedProbe, SchemaVersion, SessionStatus, SyscallAggregateResult, + SyscallAggregateSpec, SyscallAggregateValidationResult, TargetIdentity, TraceExportResult, + ValidationResult, Warning, XprobeError, }; #[derive(Debug, Parser)] @@ -248,9 +250,13 @@ struct ValidateArgs { match_policy: Option, /// Validate bounded PID-scoped CPU sampling instead of an event pair. - #[arg(long, conflicts_with_all = ["from", "to", "match_policy"])] + #[arg(long, conflicts_with_all = ["from", "to", "match_policy", "syscall_aggregate"])] cpu_sample: bool, + /// Validate bounded PID-scoped syscall aggregation instead of an event pair. + #[arg(long, conflicts_with_all = ["from", "to", "match_policy", "cpu_sample"])] + syscall_aggregate: bool, + #[command(flatten)] output: CommonOutputArgs, } @@ -258,7 +264,7 @@ struct ValidateArgs { #[derive(Debug, Args)] struct MeasureArgs { /// Versioned measurement JSON for a live target. - #[arg(long, conflicts_with_all = ["input", "pid", "from", "to", "match_policy", "cpu_sample", "samples", "duration_ms", "timeout_ms", "max_events", "max_samples", "frequency_hz", "stack_depth", "max_threads", "aggregate", "max_groups", "name"])] + #[arg(long, conflicts_with_all = ["input", "pid", "from", "to", "match_policy", "cpu_sample", "syscall_aggregate", "samples", "duration_ms", "timeout_ms", "max_events", "max_samples", "frequency_hz", "stack_depth", "max_threads", "max_inflight", "aggregate", "max_groups", "name"])] spec: Option, /// Completed CUPTI binary, host capture JSON, or Event JSONL; repeat to merge. @@ -337,6 +343,18 @@ struct MeasureArgs { )] max_threads: Option, + /// Aggregate all syscall lifecycles in bounded BPF maps. + #[arg(long, conflicts_with_all = ["input", "from", "to", "match_policy", "samples", "cpu_sample", "aggregate", "events_out", "format", "cupti_socket", "agent"])] + syscall_aggregate: bool, + + /// Bound concurrent syscall lifecycle starts retained in BPF. + #[arg( + long, + requires = "syscall_aggregate", + default_value_if("syscall_aggregate", "true", "1024") + )] + max_inflight: Option, + /// Stop after this many matched samples. #[arg(long)] samples: Option, @@ -357,7 +375,8 @@ struct MeasureArgs { #[arg( long, default_value_if("aggregate", "true", "4096"), - default_value_if("cpu_sample", "true", "256") + default_value_if("cpu_sample", "true", "256"), + default_value_if("syscall_aggregate", "true", "256") )] max_groups: Option, @@ -751,10 +770,12 @@ fn run_measure(args: MeasureArgs) -> ExitCode { to, match_policy, cpu_sample, + syscall_aggregate, frequency_hz, max_samples, stack_depth, max_threads, + max_inflight, samples, duration_ms, max_events, @@ -809,10 +830,38 @@ fn run_measure(args: MeasureArgs) -> ExitCode { json, ); } + if syscall_aggregate { + let Some(pid) = pid else { + return emit_error( + ErrorCode::InvalidEventSelector, + "--syscall-aggregate requires --pid".to_owned(), + true, + json, + ); + }; + let Some(duration_ms) = duration_ms else { + return emit_error( + ErrorCode::SessionLimitExceeded, + "--syscall-aggregate requires --duration-ms".to_owned(), + true, + json, + ); + }; + return run_direct_syscall_aggregate( + pid, + name, + duration_ms, + timeout_ms, + u64::try_from(max_groups.expect("clap supplies syscall group capacity")) + .unwrap_or(u64::MAX), + max_inflight.expect("clap supplies syscall inflight capacity"), + json, + ); + } if max_groups.is_some() && !aggregate { return emit_error( ErrorCode::InvalidEventSelector, - "--max-groups requires --aggregate or --cpu-sample".to_owned(), + "--max-groups requires --aggregate, --cpu-sample, or --syscall-aggregate".to_owned(), true, json, ); @@ -1101,6 +1150,165 @@ fn emit_cpu_inventory(result: &CpuSampleInventoryResult, json: bool) -> ExitCode ExitCode::SUCCESS } +#[allow(clippy::too_many_arguments)] +fn run_direct_syscall_aggregate( + pid: u32, + name: Option, + duration_ms: u64, + timeout_ms: u64, + max_groups: u64, + max_inflight: u64, + json: bool, +) -> ExitCode { + let report = match inspect::run(pid) { + Ok(report) => report, + Err(error) => { + return emit_error(error.code(), error.to_string(), error.recoverable(), json); + } + }; + let spec = SyscallAggregateSpec { + schema_version: SchemaVersion::current(), + name, + target: report.target.clone(), + duration_ms, + timeout_ms, + max_groups, + max_inflight, + }; + run_syscall_aggregate_spec(&spec, Some(report), json) +} + +fn run_syscall_aggregate_spec( + spec: &SyscallAggregateSpec, + inspected: Option, + json: bool, +) -> ExitCode { + let report = match inspected.map_or_else(|| inspect::run(spec.target.pid), Ok) { + Ok(report) => report, + Err(error) => { + return emit_error(error.code(), error.to_string(), error.recoverable(), json); + } + }; + if let Err(error) = inspect::verify_target(&spec.target) { + return emit_error(error.code(), error.to_string(), error.recoverable(), json); + } + let validation = match syscall_aggregate::validate(&report) { + Ok(validation) => validation, + Err(error) => { + return emit_error(error.code(), error.to_string(), error.recoverable(), json); + } + }; + if !validation.valid { + let message = validation.issues.first().map_or_else( + || "syscall aggregate requirements are not satisfied".to_owned(), + |issue| issue.message.clone(), + ); + return emit_error(ErrorCode::PermissionDenied, message, true, json); + } + let request = match syscall_collector_request(spec) { + Ok(request) => request, + Err(error) => return emit_command_failure(error, json), + }; + let capture = match syscall_collector::collect(&request) { + Ok(capture) => capture, + Err(error) => { + return emit_error(error.code(), error.to_string(), error.recoverable(), json); + } + }; + let capture = syscall_aggregate::SyscallCaptureData { + groups: capture + .groups + .into_iter() + .map(|group| syscall_aggregate::RawSyscallGroup { + syscall_number: group.syscall_number, + count: group.count, + errors: group.errors, + total_duration_ns: group.total_duration_ns, + min_duration_ns: group.min_duration_ns, + max_duration_ns: group.max_duration_ns, + }) + .collect(), + observed_entries: capture.observed_entries, + matched_exits: capture.matched_exits, + unmatched_exits: capture.unmatched_exits, + dropped_aggregates: capture.dropped_aggregates, + inflight_at_end: capture.inflight_at_end, + }; + let result = match syscall_aggregate::analyze( + &report, + spec, + format!("xp_syscalls_{}", std::process::id()), + &capture, + ) { + Ok(result) => result, + Err(error) => { + return emit_error(error.code(), error.to_string(), error.recoverable(), json); + } + }; + emit_syscall_inventory(&result, json) +} + +fn syscall_collector_request( + spec: &SyscallAggregateSpec, +) -> Result { + if spec.duration_ms == 0 || spec.max_groups == 0 || spec.max_inflight == 0 { + return Err(CommandFailure::new( + ErrorCode::SessionLimitExceeded, + "syscall aggregate duration, groups, and inflight capacity must be positive", + true, + )); + } + let max_groups = u32::try_from(spec.max_groups).map_err(|error| { + CommandFailure::new( + ErrorCode::SessionLimitExceeded, + format!("syscall max_groups exceed the BPF map ABI: {error}"), + true, + ) + })?; + let max_inflight = u32::try_from(spec.max_inflight).map_err(|error| { + CommandFailure::new( + ErrorCode::SessionLimitExceeded, + format!("syscall max_inflight exceeds the BPF map ABI: {error}"), + true, + ) + })?; + Ok(SyscallAggregateRequest { + target: spec.target.clone(), + duration: Duration::from_millis(spec.duration_ms), + timeout: Duration::from_millis(spec.timeout_ms), + max_groups, + max_inflight, + }) +} + +fn emit_syscall_inventory(result: &SyscallAggregateResult, json: bool) -> ExitCode { + if json { + println!( + "{}", + serde_json::to_string_pretty(result).expect("syscall inventory must serialize") + ); + } else { + println!( + "Syscall inventory: {} entries, {} matched exits", + result.collection.observed_entries, result.collection.matched_exits + ); + for group in result.inventory.groups.iter().take(10) { + let label = group + .syscall_name + .clone() + .unwrap_or_else(|| format!("syscall#{}", group.syscall_number)); + println!( + " {label}: count={} total={}ns mean={:.0}ns errors={}", + group.count, group.duration_ns.total, group.duration_ns.mean, group.errors + ); + } + for warning in &result.warnings { + println!("Warning: {}: {}", warning.code, warning.message); + } + } + ExitCode::SUCCESS +} + fn validate_measure_source(pid: Option, input: &[PathBuf]) -> Result<(), CommandFailure> { if pid.is_some() && !input.is_empty() { return Err(CommandFailure::new( @@ -1227,6 +1435,7 @@ fn run_completed_measurement( } } +#[allow(clippy::too_many_lines)] fn run_measure_spec( path: &Path, cupti_socket: Option, @@ -1266,6 +1475,15 @@ fn run_measure_spec( json, ); } + if value.get("max_inflight").is_some() { + return run_syscall_aggregate_file_spec( + value, + path, + cupti_socket.is_some() || agent_path.is_some(), + events_out.is_some() || format.is_some(), + json, + ); + } let spec: MeasurementSpec = match serde_json::from_slice(&bytes) { Ok(spec) => spec, Err(error) => { @@ -1363,6 +1581,35 @@ fn run_cpu_sampling_file_spec( run_cpu_sampling_spec(&spec, None, json) } +fn run_syscall_aggregate_file_spec( + value: serde_json::Value, + path: &Path, + has_device_options: bool, + has_artifact_options: bool, + json: bool, +) -> ExitCode { + if has_device_options || has_artifact_options { + return emit_error( + ErrorCode::InvalidEventSelector, + "SyscallAggregateSpec does not accept CUPTI or event artifact options".to_owned(), + true, + json, + ); + } + let spec: SyscallAggregateSpec = match serde_json::from_value(value) { + Ok(spec) => spec, + Err(error) => { + return emit_error( + ErrorCode::InvalidEventSelector, + format!("invalid SyscallAggregateSpec {}: {error}", path.display()), + true, + json, + ); + } + }; + run_syscall_aggregate_spec(&spec, None, json) +} + fn measurement_spec_capacity(spec: &MeasurementSpec) -> Result { let capacity = match spec.measurement_mode { MeasurementMode::Exact if spec.max_groups.is_some() => { @@ -3371,6 +3618,7 @@ fn run_validate(args: ValidateArgs) -> ExitCode { to, match_policy, cpu_sample, + syscall_aggregate, output: CommonOutputArgs { json, @@ -3391,10 +3639,16 @@ fn run_validate(args: ValidateArgs) -> ExitCode { Err(error) => emit_error(error.code(), error.to_string(), error.recoverable(), json), }; } + if syscall_aggregate { + return match syscall_aggregate::validate(&report) { + Ok(result) => emit_syscall_aggregate_validation(&result, json), + Err(error) => emit_error(error.code(), error.to_string(), error.recoverable(), json), + }; + } let (Some(from), Some(to), Some(match_policy)) = (from, to, match_policy) else { return emit_error( ErrorCode::InvalidEventSelector, - "validate requires --from, --to, and --match unless --cpu-sample is used".to_owned(), + "validate requires an event pair, --cpu-sample, or --syscall-aggregate".to_owned(), true, json, ); @@ -3443,6 +3697,30 @@ fn emit_cpu_sampling_validation(result: &CpuSamplingValidationResult, json: bool ExitCode::SUCCESS } +fn emit_syscall_aggregate_validation( + result: &SyscallAggregateValidationResult, + json: bool, +) -> ExitCode { + if json { + println!( + "{}", + serde_json::to_string_pretty(result) + .expect("syscall aggregate validation result must serialize") + ); + } else { + println!( + "Syscall aggregate validation: {}", + if result.valid { "valid" } else { "invalid" } + ); + println!("Target: PID {}", result.target.pid); + print_check("eBPF", &result.ebpf); + for issue in &result.issues { + println!("Issue: {}: {}", issue.code, issue.message); + } + } + ExitCode::SUCCESS +} + fn run_resolve(args: ResolveArgs) -> ExitCode { let ResolveArgs { pid, diff --git a/xprobe/cli/tests/validate.rs b/xprobe/cli/tests/validate.rs index 675718a..1eadc48 100644 --- a/xprobe/cli/tests/validate.rs +++ b/xprobe/cli/tests/validate.rs @@ -2,7 +2,7 @@ use std::process::Command; use xprobe_protocol::{ AgentActivation, CpuSamplingValidationResult, ErrorCode, ErrorResponse, MatchPolicy, - PythonSymbolizationStatus, ValidationResult, + PythonSymbolizationStatus, SyscallAggregateValidationResult, ValidationResult, }; #[test] @@ -113,3 +113,26 @@ fn validate_accepts_pid_scoped_cpu_sampling_without_ebpf() { assert!(result.target_threads >= 1); assert_eq!(result.python_status, PythonSymbolizationStatus::NotDetected); } + +#[test] +fn validate_reports_syscall_aggregate_ebpf_requirements() { + let output = Command::new(env!("CARGO_BIN_EXE_xprobe")) + .args([ + "validate", + "--pid", + &std::process::id().to_string(), + "--syscall-aggregate", + "--json", + "--non-interactive", + "--no-color", + ]) + .output() + .expect("xprobe validate must run"); + + assert!(output.status.success()); + let result: SyscallAggregateValidationResult = + serde_json::from_slice(&output.stdout).expect("stdout must contain validation JSON"); + assert!(result.requirements.needs_ebpf); + assert!(!result.requirements.target_mutation); + assert_eq!(result.valid, result.issues.is_empty()); +} diff --git a/xprobe/collector/src/lib.rs b/xprobe/collector/src/lib.rs index 6f4a23f..758daff 100644 --- a/xprobe/collector/src/lib.rs +++ b/xprobe/collector/src/lib.rs @@ -4,4 +4,5 @@ pub mod completed; pub mod cpu_sampling; pub mod cupti; pub mod linux; +pub mod syscall_aggregate; pub mod uprobe; diff --git a/xprobe/collector/src/syscall_aggregate.rs b/xprobe/collector/src/syscall_aggregate.rs new file mode 100644 index 0000000..907f623 --- /dev/null +++ b/xprobe/collector/src/syscall_aggregate.rs @@ -0,0 +1,471 @@ +use std::{ + error::Error, + ffi::OsStr, + fmt, fs, io, + os::unix::fs::MetadataExt, + path::PathBuf, + thread, + time::{Duration, Instant}, +}; + +use libbpf_rs::{ErrorKind, Link, MapCore, MapFlags, Object, ObjectBuilder}; +use xprobe_protocol::{ErrorCode, TargetIdentity}; + +const WAIT_INTERVAL: Duration = Duration::from_millis(10); +const BPF_OBJECT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/xprobe.bpf.o")); + +#[derive(Debug, Clone)] +pub struct SyscallAggregateRequest { + pub target: TargetIdentity, + pub duration: Duration, + pub timeout: Duration, + pub max_groups: u32, + pub max_inflight: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RawSyscallAggregate { + pub syscall_number: u32, + pub count: u64, + pub errors: u64, + pub total_duration_ns: u64, + pub min_duration_ns: u64, + pub max_duration_ns: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SyscallAggregateCapture { + pub groups: Vec, + pub observed_entries: u64, + pub matched_exits: u64, + pub unmatched_exits: u64, + pub dropped_aggregates: u64, + pub inflight_at_end: u64, +} + +#[derive(Debug)] +pub enum SyscallAggregateError { + InvalidRequest(&'static str), + TargetNamespace { + path: PathBuf, + source: io::Error, + }, + MissingObjectMember { + kind: &'static str, + name: String, + }, + Libbpf { + operation: &'static str, + source: libbpf_rs::Error, + }, + MalformedMapValue { + map: &'static str, + expected: usize, + actual: usize, + }, + Timeout, +} + +impl SyscallAggregateError { + #[must_use] + pub fn code(&self) -> ErrorCode { + match self { + Self::InvalidRequest(_) | Self::Timeout => ErrorCode::SessionLimitExceeded, + Self::TargetNamespace { source, .. } + if source.kind() == io::ErrorKind::PermissionDenied => + { + ErrorCode::PermissionDenied + } + Self::TargetNamespace { source, .. } if source.kind() == io::ErrorKind::NotFound => { + ErrorCode::TargetExited + } + Self::Libbpf { + operation: "disarm syscall aggregate", + .. + } => ErrorCode::CleanupFailed, + Self::Libbpf { source, .. } if source.kind() == ErrorKind::PermissionDenied => { + ErrorCode::PermissionDenied + } + Self::TargetNamespace { .. } + | Self::MissingObjectMember { .. } + | Self::Libbpf { .. } + | Self::MalformedMapValue { .. } => ErrorCode::Internal, + } + } + + #[must_use] + pub fn recoverable(&self) -> bool { + matches!( + self.code(), + ErrorCode::SessionLimitExceeded | ErrorCode::PermissionDenied | ErrorCode::TargetExited + ) + } +} + +impl fmt::Display for SyscallAggregateError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidRequest(message) => { + write!(formatter, "invalid syscall aggregate request: {message}") + } + Self::TargetNamespace { path, source } => write!( + formatter, + "failed to inspect target PID namespace at {}: {source}", + path.display() + ), + Self::MissingObjectMember { kind, name } => { + write!(formatter, "BPF object is missing {kind} {name}") + } + Self::Libbpf { operation, source } => { + write!(formatter, "failed to {operation}: {source:#}") + } + Self::MalformedMapValue { + map, + expected, + actual, + } => write!( + formatter, + "BPF map {map} value has size {actual}, expected {expected}" + ), + Self::Timeout => formatter.write_str("syscall aggregate collection timed out"), + } + } +} + +impl Error for SyscallAggregateError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::TargetNamespace { source, .. } => Some(source), + Self::Libbpf { source, .. } => Some(source), + Self::InvalidRequest(_) + | Self::MissingObjectMember { .. } + | Self::MalformedMapValue { .. } + | Self::Timeout => None, + } + } +} + +/// Collect bounded PID-scoped syscall lifecycle aggregates entirely in BPF maps. +/// +/// # Errors +/// +/// Returns [`SyscallAggregateError`] for invalid bounds, BPF setup, attachment, +/// timeout, cleanup, or map decoding failures. +pub fn collect( + request: &SyscallAggregateRequest, +) -> Result { + validate_request(request)?; + let started = Instant::now(); + let timeout_at = + started + .checked_add(request.timeout) + .ok_or(SyscallAggregateError::InvalidRequest( + "timeout overflows the monotonic clock", + ))?; + let object = load_object(request.max_groups, request.max_inflight)?; + configure_object(&object, request)?; + let _links = attach_programs(&object)?; + set_armed(&object, true)?; + let capture_end = Instant::now().checked_add(request.duration).ok_or( + SyscallAggregateError::InvalidRequest("duration overflows the monotonic clock"), + )?; + while Instant::now() < capture_end { + if Instant::now() >= timeout_at { + set_armed(&object, false)?; + return Err(SyscallAggregateError::Timeout); + } + thread::sleep(WAIT_INTERVAL.min(capture_end.saturating_duration_since(Instant::now()))); + } + set_armed(&object, false)?; + read_capture(&object) +} + +fn validate_request(request: &SyscallAggregateRequest) -> Result<(), SyscallAggregateError> { + if request.duration.is_zero() { + return Err(SyscallAggregateError::InvalidRequest( + "duration must be positive", + )); + } + if request.timeout < request.duration { + return Err(SyscallAggregateError::InvalidRequest( + "timeout must be at least the capture duration", + )); + } + if request.max_groups == 0 { + return Err(SyscallAggregateError::InvalidRequest( + "max_groups must be positive", + )); + } + if request.max_inflight == 0 { + return Err(SyscallAggregateError::InvalidRequest( + "max_inflight must be positive", + )); + } + Ok(()) +} + +fn load_object(max_groups: u32, max_inflight: u32) -> Result { + let mut builder = ObjectBuilder::default(); + builder + .name("xprobe_syscalls") + .map_err(|source| libbpf_error("name BPF object", source))?; + let mut open_object = builder + .open_memory(BPF_OBJECT) + .map_err(|source| libbpf_error("open BPF object", source))?; + open_object + .maps_mut() + .find(|map| map.name() == OsStr::new("syscall_aggregate_groups")) + .ok_or_else(|| missing("map", "syscall_aggregate_groups"))? + .set_max_entries(max_groups) + .map_err(|source| libbpf_error("size syscall aggregate groups", source))?; + open_object + .maps_mut() + .find(|map| map.name() == OsStr::new("syscall_aggregate_inflight")) + .ok_or_else(|| missing("map", "syscall_aggregate_inflight"))? + .set_max_entries(max_inflight) + .map_err(|source| libbpf_error("size syscall inflight table", source))?; + open_object + .load() + .map_err(|source| libbpf_error("load BPF object", source)) +} + +fn configure_object( + object: &Object, + request: &SyscallAggregateRequest, +) -> Result<(), SyscallAggregateError> { + let namespace_path = PathBuf::from(format!("/proc/{}/ns/pid", request.target.pid)); + let namespace = + fs::metadata(&namespace_path).map_err(|source| SyscallAggregateError::TargetNamespace { + path: namespace_path, + source, + })?; + let mut config = [0_u8; 24]; + config[0..8].copy_from_slice(&namespace.dev().to_ne_bytes()); + config[8..16].copy_from_slice(&namespace.ino().to_ne_bytes()); + config[16..20].copy_from_slice(&request.target.pid.to_ne_bytes()); + let key = 0_u32.to_ne_bytes(); + object + .maps() + .find(|map| map.name() == OsStr::new("syscall_aggregate_config")) + .ok_or_else(|| missing("map", "syscall_aggregate_config"))? + .update(&key, &config, MapFlags::ANY) + .map_err(|source| libbpf_error("configure syscall aggregate", source)) +} + +fn attach_programs(object: &Object) -> Result, SyscallAggregateError> { + let mut links = Vec::with_capacity(2); + for (program_name, tracepoint_name) in [ + ("xprobe_aggregate_syscall_entry", "sys_enter"), + ("xprobe_aggregate_syscall_exit", "sys_exit"), + ] { + let program = object + .progs_mut() + .find(|program| program.name() == OsStr::new(program_name)) + .ok_or_else(|| missing("program", program_name))?; + links.push( + program + .attach_raw_tracepoint(tracepoint_name) + .map_err(|source| libbpf_error("attach syscall aggregate tracepoint", source))?, + ); + } + Ok(links) +} + +fn set_armed(object: &Object, armed: bool) -> Result<(), SyscallAggregateError> { + let key = 0_u32.to_ne_bytes(); + let map = object + .maps() + .find(|map| map.name() == OsStr::new("syscall_aggregate_config")) + .ok_or_else(|| missing("map", "syscall_aggregate_config"))?; + let mut config = map + .lookup(&key, MapFlags::ANY) + .map_err(|source| libbpf_error("read syscall aggregate config", source))? + .ok_or_else(|| missing("map value", "syscall_aggregate_config"))?; + if config.len() != 24 { + return Err(SyscallAggregateError::MalformedMapValue { + map: "syscall_aggregate_config", + expected: 24, + actual: config.len(), + }); + } + config[20..24].copy_from_slice(&u32::from(armed).to_ne_bytes()); + let operation = if armed { + "arm syscall aggregate" + } else { + "disarm syscall aggregate" + }; + map.update(&key, &config, MapFlags::ANY) + .map_err(|source| libbpf_error(operation, source)) +} + +fn read_capture(object: &Object) -> Result { + let map = object + .maps() + .find(|map| map.name() == OsStr::new("syscall_aggregate_groups")) + .ok_or_else(|| missing("map", "syscall_aggregate_groups"))?; + let mut groups = Vec::new(); + for key in map.keys() { + let syscall_number = decode_u32("syscall_aggregate_groups", &key)?; + let values = map + .lookup_percpu(&key, MapFlags::ANY) + .map_err(|source| libbpf_error("read syscall aggregate group", source))? + .ok_or_else(|| missing("map value", "syscall_aggregate_groups"))?; + groups.push(sum_group(syscall_number, &values)?); + } + groups.sort_by(|left, right| { + right + .total_duration_ns + .cmp(&left.total_duration_ns) + .then_with(|| left.syscall_number.cmp(&right.syscall_number)) + }); + let summary = read_summary(object)?; + let inflight_at_end = u64::try_from( + object + .maps() + .find(|map| map.name() == OsStr::new("syscall_aggregate_inflight")) + .ok_or_else(|| missing("map", "syscall_aggregate_inflight"))? + .keys() + .count(), + ) + .unwrap_or(u64::MAX); + Ok(SyscallAggregateCapture { + groups, + observed_entries: summary[0], + matched_exits: summary[1], + unmatched_exits: summary[2], + dropped_aggregates: summary[3], + inflight_at_end, + }) +} + +fn sum_group( + syscall_number: u32, + values: &[Vec], +) -> Result { + let mut group = RawSyscallAggregate { + syscall_number, + count: 0, + errors: 0, + total_duration_ns: 0, + min_duration_ns: u64::MAX, + max_duration_ns: 0, + }; + for value in values { + let fields = decode_u64_fields::<5>("syscall_aggregate_groups", value)?; + if fields[0] == 0 { + continue; + } + group.count = group.count.saturating_add(fields[0]); + group.errors = group.errors.saturating_add(fields[1]); + group.total_duration_ns = group.total_duration_ns.saturating_add(fields[2]); + group.min_duration_ns = group.min_duration_ns.min(fields[3]); + group.max_duration_ns = group.max_duration_ns.max(fields[4]); + } + if group.count == 0 { + group.min_duration_ns = 0; + } + Ok(group) +} + +fn read_summary(object: &Object) -> Result<[u64; 4], SyscallAggregateError> { + let key = 0_u32.to_ne_bytes(); + let value = object + .maps() + .find(|map| map.name() == OsStr::new("syscall_aggregate_summary")) + .ok_or_else(|| missing("map", "syscall_aggregate_summary"))? + .lookup(&key, MapFlags::ANY) + .map_err(|source| libbpf_error("read syscall aggregate summary", source))? + .ok_or_else(|| missing("map value", "syscall_aggregate_summary"))?; + decode_u64_fields::<4>("syscall_aggregate_summary", &value) +} + +fn decode_u32(map: &'static str, bytes: &[u8]) -> Result { + let bytes: [u8; 4] = + bytes + .try_into() + .map_err(|_| SyscallAggregateError::MalformedMapValue { + map, + expected: 4, + actual: bytes.len(), + })?; + Ok(u32::from_ne_bytes(bytes)) +} + +fn decode_u64_fields( + map: &'static str, + bytes: &[u8], +) -> Result<[u64; N], SyscallAggregateError> { + let expected = N * 8; + if bytes.len() != expected { + return Err(SyscallAggregateError::MalformedMapValue { + map, + expected, + actual: bytes.len(), + }); + } + Ok(std::array::from_fn(|index| { + let start = index * 8; + u64::from_ne_bytes( + bytes[start..start + 8] + .try_into() + .expect("checked u64 field"), + ) + })) +} + +fn missing(kind: &'static str, name: &str) -> SyscallAggregateError { + SyscallAggregateError::MissingObjectMember { + kind, + name: name.to_owned(), + } +} + +fn libbpf_error(operation: &'static str, source: libbpf_rs::Error) -> SyscallAggregateError { + SyscallAggregateError::Libbpf { operation, source } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sums_per_cpu_aggregate_values() { + let values = vec![fields(&[3, 1, 30, 5, 15]), fields(&[2, 0, 50, 20, 30])]; + assert_eq!( + sum_group(9, &values).unwrap(), + RawSyscallAggregate { + syscall_number: 9, + count: 5, + errors: 1, + total_duration_ns: 80, + min_duration_ns: 5, + max_duration_ns: 30, + } + ); + } + + #[test] + fn rejects_zero_map_capacities() { + let request = SyscallAggregateRequest { + target: TargetIdentity { + pid: 1, + process_start_time: 1, + }, + duration: Duration::from_millis(1), + timeout: Duration::from_millis(2), + max_groups: 0, + max_inflight: 1, + }; + assert!(matches!( + validate_request(&request), + Err(SyscallAggregateError::InvalidRequest(_)) + )); + } + + fn fields(values: &[u64]) -> Vec { + values + .iter() + .flat_map(|value| value.to_ne_bytes()) + .collect() + } +} diff --git a/xprobe/core/src/lib.rs b/xprobe/core/src/lib.rs index 67dbebb..39b4259 100644 --- a/xprobe/core/src/lib.rs +++ b/xprobe/core/src/lib.rs @@ -8,4 +8,5 @@ pub mod doctor; pub mod inject; pub mod inspect; pub mod resolve; +pub mod syscall_aggregate; pub mod validate; diff --git a/xprobe/core/src/syscall_aggregate.rs b/xprobe/core/src/syscall_aggregate.rs new file mode 100644 index 0000000..c428c24 --- /dev/null +++ b/xprobe/core/src/syscall_aggregate.rs @@ -0,0 +1,371 @@ +use std::{collections::BTreeMap, error::Error, fmt}; + +use xprobe_protocol::{ + AggregateDuration, ErrorCode, ProcessReport, SchemaVersion, SessionStatus, + SyscallAggregateCollectionSummary, SyscallAggregateCompleteness, SyscallAggregateGroup, + SyscallAggregateInventory, SyscallAggregateRequirements, SyscallAggregateResult, + SyscallAggregateSpec, SyscallAggregateValidationResult, ValidationIssue, Warning, +}; + +use crate::{ + doctor::{self, DoctorError}, + inspect::{self, InspectError}, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RawSyscallGroup { + pub syscall_number: u32, + pub count: u64, + pub errors: u64, + pub total_duration_ns: u64, + pub min_duration_ns: u64, + pub max_duration_ns: u64, +} + +#[derive(Debug, Clone)] +pub struct SyscallCaptureData { + pub groups: Vec, + pub observed_entries: u64, + pub matched_exits: u64, + pub unmatched_exits: u64, + pub dropped_aggregates: u64, + pub inflight_at_end: u64, +} + +#[derive(Debug)] +pub enum SyscallAggregateError { + Inspect(InspectError), + Doctor(DoctorError), +} + +impl SyscallAggregateError { + #[must_use] + pub const fn code(&self) -> ErrorCode { + match self { + Self::Inspect(error) => error.code(), + Self::Doctor(_) => ErrorCode::Internal, + } + } + + #[must_use] + pub const fn recoverable(&self) -> bool { + match self { + Self::Inspect(error) => error.recoverable(), + Self::Doctor(_) => false, + } + } +} + +impl fmt::Display for SyscallAggregateError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Inspect(error) => error.fmt(formatter), + Self::Doctor(error) => write!( + formatter, + "syscall aggregate capability check failed: {error}" + ), + } + } +} + +impl Error for SyscallAggregateError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Inspect(error) => Some(error), + Self::Doctor(error) => Some(error), + } + } +} + +impl From for SyscallAggregateError { + fn from(error: InspectError) -> Self { + Self::Inspect(error) + } +} + +impl From for SyscallAggregateError { + fn from(error: DoctorError) -> Self { + Self::Doctor(error) + } +} + +/// Validate read-only requirements for PID-scoped syscall aggregation. +/// +/// # Errors +/// +/// Returns [`SyscallAggregateError`] if target identity or local capability +/// inspection fails. +pub fn validate( + report: &ProcessReport, +) -> Result { + inspect::verify_target(&report.target)?; + let capabilities = doctor::run()?; + let ebpf = capabilities.checks.ebpf_permissions.clone(); + let mut issues = Vec::new(); + if !report.capabilities.tracepoint { + issues.push(ValidationIssue { + code: ErrorCode::PermissionDenied, + message: "syscall aggregation requires effective eBPF tracepoint privileges".to_owned(), + }); + } + inspect::verify_target(&report.target)?; + Ok(SyscallAggregateValidationResult { + schema_version: SchemaVersion::current(), + ok: true, + valid: issues.is_empty(), + target: report.target.clone(), + requirements: SyscallAggregateRequirements { + needs_ebpf: true, + target_mutation: false, + }, + ebpf, + issues, + warnings: Vec::new(), + }) +} + +/// Normalize one completed BPF syscall aggregate capture. +/// +/// # Errors +/// +/// Returns [`SyscallAggregateError`] when the target identity changed around +/// collection. +pub fn analyze( + report: &ProcessReport, + spec: &SyscallAggregateSpec, + session_id: String, + capture: &SyscallCaptureData, +) -> Result { + inspect::verify_target(&report.target)?; + let mut groups = capture + .groups + .iter() + .filter(|group| group.count != 0) + .map(normalize_group) + .collect::>(); + groups.sort_by(|left, right| { + right + .duration_ns + .total + .cmp(&left.duration_ns.total) + .then_with(|| right.count.cmp(&left.count)) + .then_with(|| left.syscall_number.cmp(&right.syscall_number)) + }); + let complete = capture.unmatched_exits == 0 + && capture.inflight_at_end == 0 + && capture.dropped_aggregates == 0; + let mut warnings = Vec::new(); + if capture.dropped_aggregates != 0 { + warnings.push(warning( + "SYSCALL_AGGREGATES_DROPPED", + &format!( + "{} syscall lifecycles could not be retained; increase max_groups or max_inflight", + capture.dropped_aggregates + ), + )); + } + if capture.unmatched_exits != 0 || capture.inflight_at_end != 0 { + warnings.push(warning( + "SYSCALL_LIFECYCLES_UNMATCHED", + &format!( + "{} exits had no start and {} starts remained open at capture end", + capture.unmatched_exits, capture.inflight_at_end + ), + )); + } + inspect::verify_target(&report.target)?; + Ok(SyscallAggregateResult { + schema_version: SchemaVersion::current(), + ok: true, + session_id, + status: SessionStatus::Completed, + target: report.target.clone(), + inventory: SyscallAggregateInventory { + name: spec.name.clone(), + duration_ms: spec.duration_ms, + groups, + }, + collection: SyscallAggregateCollectionSummary { + completeness: if complete { + SyscallAggregateCompleteness::Complete + } else { + SyscallAggregateCompleteness::Incomplete + }, + observed_entries: capture.observed_entries, + matched_exits: capture.matched_exits, + unmatched_exits: capture.unmatched_exits, + inflight_at_end: capture.inflight_at_end, + dropped_aggregates: capture.dropped_aggregates, + group_capacity: spec.max_groups, + groups: u64::try_from(capture.groups.len()).unwrap_or(u64::MAX), + table_utilization: ratio( + u64::try_from(capture.groups.len()).unwrap_or(u64::MAX), + spec.max_groups.max(1), + ), + }, + warnings, + }) +} + +fn normalize_group(group: &RawSyscallGroup) -> SyscallAggregateGroup { + let syscall_name = syscall_name(group.syscall_number).map(str::to_owned); + let (entry_selector_hint, exit_selector_hint) = + syscall_name.as_ref().map_or((None, None), |name| { + ( + Some(format!("syscall:{name}:entry")), + Some(format!("syscall:{name}:exit")), + ) + }); + SyscallAggregateGroup { + syscall_number: group.syscall_number, + syscall_name, + count: group.count, + errors: group.errors, + duration_ns: AggregateDuration { + min: group.min_duration_ns, + mean: ratio(group.total_duration_ns, group.count), + max: group.max_duration_ns, + total: group.total_duration_ns, + }, + entry_selector_hint, + exit_selector_hint, + } +} + +fn syscall_name(number: u32) -> Option<&'static str> { + Some(match number { + 0 => "read", + 1 => "write", + 3 => "close", + 5 => "fstat", + 7 => "poll", + 8 => "lseek", + 9 => "mmap", + 10 => "mprotect", + 11 => "munmap", + 12 => "brk", + 16 => "ioctl", + 17 => "pread64", + 18 => "pwrite64", + 19 => "readv", + 20 => "writev", + 24 => "sched_yield", + 25 => "mremap", + 26 => "msync", + 27 => "mincore", + 28 => "madvise", + 35 => "nanosleep", + 39 => "getpid", + 41 => "socket", + 42 => "connect", + 43 => "accept", + 44 => "sendto", + 45 => "recvfrom", + 46 => "sendmsg", + 47 => "recvmsg", + 56 => "clone", + 57 => "fork", + 58 => "vfork", + 59 => "execve", + 60 => "exit", + 61 => "wait4", + 72 => "fcntl", + 73 => "flock", + 74 => "fsync", + 75 => "fdatasync", + 78 => "getdents", + 186 => "gettid", + 202 => "futex", + 203 => "sched_setaffinity", + 204 => "sched_getaffinity", + 232 => "epoll_wait", + 233 => "epoll_ctl", + 234 => "tgkill", + 257 => "openat", + 262 => "newfstatat", + 270 => "pselect6", + 271 => "ppoll", + 272 => "unshare", + 275 => "splice", + 281 => "epoll_pwait", + 288 => "accept4", + 290 => "eventfd2", + 291 => "epoll_create1", + 292 => "dup3", + 293 => "pipe2", + 295 => "preadv", + 296 => "pwritev", + 298 => "perf_event_open", + 299 => "recvmmsg", + 302 => "prlimit64", + 307 => "sendmmsg", + 309 => "getcpu", + 310 => "process_vm_readv", + 311 => "process_vm_writev", + 314 => "sched_setattr", + 315 => "sched_getattr", + 317 => "seccomp", + 318 => "getrandom", + 319 => "memfd_create", + 321 => "bpf", + 322 => "execveat", + 323 => "userfaultfd", + 324 => "membarrier", + 325 => "mlock2", + 326 => "copy_file_range", + 327 => "preadv2", + 328 => "pwritev2", + 332 => "statx", + 334 => "rseq", + 424 => "pidfd_send_signal", + 425 => "io_uring_setup", + 426 => "io_uring_enter", + 427 => "io_uring_register", + 434 => "pidfd_open", + 435 => "clone3", + 436 => "close_range", + 437 => "openat2", + 438 => "pidfd_getfd", + 439 => "faccessat2", + 440 => "process_madvise", + 441 => "epoll_pwait2", + 449 => "futex_waitv", + _ => return None, + }) +} + +fn warning(code: &str, message: &str) -> Warning { + Warning { + code: code.to_owned(), + message: message.to_owned(), + details: BTreeMap::new(), + } +} + +#[allow(clippy::cast_precision_loss)] +fn ratio(numerator: u64, denominator: u64) -> f64 { + numerator as f64 / denominator as f64 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalizes_known_syscalls_and_selector_hints() { + let group = normalize_group(&RawSyscallGroup { + syscall_number: 9, + count: 4, + errors: 1, + total_duration_ns: 100, + min_duration_ns: 10, + max_duration_ns: 50, + }); + assert_eq!(group.syscall_name.as_deref(), Some("mmap")); + assert!((group.duration_ns.mean - 25.0).abs() < f64::EPSILON); + assert_eq!( + group.entry_selector_hint.as_deref(), + Some("syscall:mmap:entry") + ); + } +} diff --git a/xprobe/protocol/src/lib.rs b/xprobe/protocol/src/lib.rs index 18ac4bf..97d28dd 100644 --- a/xprobe/protocol/src/lib.rs +++ b/xprobe/protocol/src/lib.rs @@ -11,6 +11,7 @@ mod measurement; mod process; mod resolve; pub mod schema; +mod syscall_aggregate; mod validate; mod version; @@ -40,6 +41,11 @@ pub use measurement::{ }; pub use process::{CgroupEntry, ProcessCredentials, ProcessCudaState, ProcessReport}; pub use resolve::{ElfObjectKind, ProcessMapping, ResolvedProbe}; +pub use syscall_aggregate::{ + SyscallAggregateCollectionSummary, SyscallAggregateCompleteness, SyscallAggregateGroup, + SyscallAggregateInventory, SyscallAggregateRequirements, SyscallAggregateResult, + SyscallAggregateSpec, SyscallAggregateValidationResult, +}; pub use validate::{ AgentActivation, EndpointSource, PolicyRecommendation, PolicyRecommendationReason, ResolvedCudaSelector, ResolvedLinuxSelector, ValidatedEndpoint, ValidationIssue, diff --git a/xprobe/protocol/src/schema.rs b/xprobe/protocol/src/schema.rs index e6f96ac..56a7f0c 100644 --- a/xprobe/protocol/src/schema.rs +++ b/xprobe/protocol/src/schema.rs @@ -3,12 +3,12 @@ use schemars::{Schema, schema_for}; use crate::{ AggregateInventoryResult, CapabilityReport, CpuSampleInventoryResult, CpuSamplingSpec, CpuSamplingValidationResult, DiscoveryResult, ErrorResponse, Event, HostCaptureResult, - MeasurementResult, MeasurementSpec, ProcessReport, ResolvedProbe, TraceExportResult, - ValidationResult, + MeasurementResult, MeasurementSpec, ProcessReport, ResolvedProbe, SyscallAggregateResult, + SyscallAggregateSpec, SyscallAggregateValidationResult, TraceExportResult, ValidationResult, }; #[must_use] -pub fn generated_schemas() -> [(&'static str, Schema); 15] { +pub fn generated_schemas() -> [(&'static str, Schema); 18] { [ ("event.schema.json", schema_for!(Event)), ("error.schema.json", schema_for!(ErrorResponse)), @@ -33,6 +33,18 @@ pub fn generated_schemas() -> [(&'static str, Schema); 15] { "cpu-sampling-validate.schema.json", schema_for!(CpuSamplingValidationResult), ), + ( + "syscall-aggregate-spec.schema.json", + schema_for!(SyscallAggregateSpec), + ), + ( + "syscall-aggregate-result.schema.json", + schema_for!(SyscallAggregateResult), + ), + ( + "syscall-aggregate-validate.schema.json", + schema_for!(SyscallAggregateValidationResult), + ), ("capability.schema.json", schema_for!(CapabilityReport)), ("discover.schema.json", schema_for!(DiscoveryResult)), ("inspect.schema.json", schema_for!(ProcessReport)), diff --git a/xprobe/protocol/src/syscall_aggregate.rs b/xprobe/protocol/src/syscall_aggregate.rs new file mode 100644 index 0000000..10f79b8 --- /dev/null +++ b/xprobe/protocol/src/syscall_aggregate.rs @@ -0,0 +1,96 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::{ + AggregateDuration, CheckResult, SchemaVersion, SessionStatus, TargetIdentity, ValidationIssue, + Warning, +}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SyscallAggregateSpec { + pub schema_version: SchemaVersion, + pub name: Option, + pub target: TargetIdentity, + pub duration_ms: u64, + pub timeout_ms: u64, + pub max_groups: u64, + pub max_inflight: u64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SyscallAggregateGroup { + pub syscall_number: u32, + pub syscall_name: Option, + pub count: u64, + pub errors: u64, + pub duration_ns: AggregateDuration, + pub entry_selector_hint: Option, + pub exit_selector_hint: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SyscallAggregateInventory { + pub name: Option, + pub duration_ms: u64, + pub groups: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum SyscallAggregateCompleteness { + Complete, + Incomplete, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SyscallAggregateCollectionSummary { + pub completeness: SyscallAggregateCompleteness, + pub observed_entries: u64, + pub matched_exits: u64, + pub unmatched_exits: u64, + pub inflight_at_end: u64, + pub dropped_aggregates: u64, + pub group_capacity: u64, + pub groups: u64, + pub table_utilization: f64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SyscallAggregateResult { + pub schema_version: SchemaVersion, + pub ok: bool, + pub session_id: String, + pub status: SessionStatus, + pub target: TargetIdentity, + pub inventory: SyscallAggregateInventory, + pub collection: SyscallAggregateCollectionSummary, + #[serde(default)] + pub warnings: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SyscallAggregateRequirements { + pub needs_ebpf: bool, + pub target_mutation: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SyscallAggregateValidationResult { + pub schema_version: SchemaVersion, + pub ok: bool, + pub valid: bool, + pub target: TargetIdentity, + pub requirements: SyscallAggregateRequirements, + pub ebpf: CheckResult, + #[serde(default)] + pub issues: Vec, + #[serde(default)] + pub warnings: Vec, +} diff --git a/xprobe/protocol/tests/contracts.rs b/xprobe/protocol/tests/contracts.rs index 3b91070..3e20e90 100644 --- a/xprobe/protocol/tests/contracts.rs +++ b/xprobe/protocol/tests/contracts.rs @@ -5,8 +5,9 @@ use serde_json::{Value, json}; use xprobe_protocol::{ AggregateInventoryResult, CapabilityReport, CpuSampleInventoryResult, CpuSamplingSpec, CpuSamplingValidationResult, DiscoveryResult, ErrorResponse, Event, HostCaptureResult, - MeasurementResult, MeasurementSpec, ProcessReport, ResolvedProbe, TraceExportResult, - ValidationResult, schema::generated_schemas, + MeasurementResult, MeasurementSpec, ProcessReport, ResolvedProbe, SyscallAggregateResult, + SyscallAggregateSpec, SyscallAggregateValidationResult, TraceExportResult, ValidationResult, + schema::generated_schemas, }; fn assert_round_trip(fixture: &Value) @@ -328,6 +329,69 @@ fn cpu_sampling_validation_contract_round_trips() { })); } +#[test] +fn syscall_aggregate_spec_contract_round_trips() { + assert_round_trip::(&json!({ + "schema_version": "2.0", + "name": "syscalls", + "target": {"pid": 1234, "process_start_time": 42}, + "duration_ms": 1000, + "timeout_ms": 30000, + "max_groups": 256, + "max_inflight": 1024 + })); +} + +#[test] +fn syscall_aggregate_result_contract_round_trips() { + assert_round_trip::(&json!({ + "schema_version": "2.0", + "ok": true, + "session_id": "xp_syscalls", + "status": "completed", + "target": {"pid": 1234, "process_start_time": 42}, + "inventory": { + "name": "syscalls", + "duration_ms": 1000, + "groups": [{ + "syscall_number": 202, + "syscall_name": "futex", + "count": 80, + "errors": 2, + "duration_ns": {"min": 100, "mean": 250.0, "max": 1000, "total": 20000}, + "entry_selector_hint": "syscall:futex:entry", + "exit_selector_hint": "syscall:futex:exit" + }] + }, + "collection": { + "completeness": "complete", + "observed_entries": 100, + "matched_exits": 100, + "unmatched_exits": 0, + "inflight_at_end": 0, + "dropped_aggregates": 0, + "group_capacity": 256, + "groups": 8, + "table_utilization": 0.03125 + }, + "warnings": [] + })); +} + +#[test] +fn syscall_aggregate_validation_contract_round_trips() { + assert_round_trip::(&json!({ + "schema_version": "2.0", + "ok": true, + "valid": true, + "target": {"pid": 1234, "process_start_time": 42}, + "requirements": {"needs_ebpf": true, "target_mutation": false}, + "ebpf": {"status": "available", "detail": "CAP_BPF and CAP_PERFMON"}, + "issues": [], + "warnings": [] + })); +} + #[test] fn trace_export_contract_round_trips() { assert_round_trip::(&json!({ From 54b82c1870f1afd55876d3969cc19294afdd1d56 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 1 Aug 2026 05:01:21 +0800 Subject: [PATCH 05/15] =?UTF-8?q?=E2=9C=A8=20feat:=20measure=20CPython=20G?= =?UTF-8?q?C=20USDT=20lifecycles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bpf/xprobe.bpf.c | 33 ++++ schemas/event.schema.json | 2 + schemas/host-capture.schema.json | 2 + schemas/measurement-result.schema.json | 2 + schemas/validate.schema.json | 14 ++ tests/CMakeLists.txt | 4 + tests/fixtures/python_gc_usdt_target.c | 38 ++++ tests/integration/run-linux-live.sh | 25 ++- tests/integration/test_linux.py | 25 ++- xprobe/collector/src/linux.rs | 106 ++++++++++- xprobe/core/src/lib.rs | 1 + xprobe/core/src/usdt.rs | 253 +++++++++++++++++++++++++ xprobe/core/src/validate.rs | 108 ++++++++++- xprobe/correlator/src/lib.rs | 165 ++++++++++++++-- xprobe/exporter/src/lib.rs | 2 + xprobe/protocol/src/event.rs | 2 + xprobe/protocol/src/validate.rs | 4 + xprobe/protocol/tests/contracts.rs | 19 +- 18 files changed, 765 insertions(+), 40 deletions(-) create mode 100644 tests/fixtures/python_gc_usdt_target.c create mode 100644 xprobe/core/src/usdt.rs diff --git a/bpf/xprobe.bpf.c b/bpf/xprobe.bpf.c index ac152cf..768e04e 100644 --- a/bpf/xprobe.bpf.c +++ b/bpf/xprobe.bpf.c @@ -87,6 +87,14 @@ struct xprobe_syscall_aggregate_summary { __u64 dropped_aggregates; }; +/* libbpf populates these maps while attaching SEC("usdt") programs. xprobe + * does not read USDT arguments, but the value layout must match usdt.bpf.h. */ +struct xprobe_usdt_spec { + __u8 arguments[12][16]; + __u64 cookie; + __s16 argument_count; +}; + struct xprobe_raw_tracepoint_context { __u64 arguments[2]; }; @@ -195,6 +203,20 @@ struct { __type(value, struct xprobe_syscall_aggregate_summary); } syscall_aggregate_summary SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 16); + __type(key, int); + __type(value, struct xprobe_usdt_spec); +} __bpf_usdt_specs SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 64); + __type(key, long); + __type(value, __u32); +} __bpf_usdt_ip_to_spec_id SEC(".maps"); + SEC("uprobe") int xprobe_handle_uprobe(void *context) { @@ -292,6 +314,17 @@ XPROBE_TRACEPOINT_PROGRAM(2) XPROBE_RAW_TRACEPOINT_PROGRAM(1) XPROBE_RAW_TRACEPOINT_PROGRAM(2) +#define XPROBE_USDT_PROGRAM(slot) \ + SEC("usdt") \ + int xprobe_handle_usdt_##slot(void *context) \ + { \ + (void)context; \ + return xprobe_emit_linux_event(slot, 0); \ + } + +XPROBE_USDT_PROGRAM(1) +XPROBE_USDT_PROGRAM(2) + static __attribute__((always_inline)) __u32 xprobe_syscall_probe_id(const __s64 numbers[2], __s64 syscall_number) { diff --git a/schemas/event.schema.json b/schemas/event.schema.json index c19d114..f98e1b8 100644 --- a/schemas/event.schema.json +++ b/schemas/event.schema.json @@ -339,6 +339,8 @@ "gpu_memset_end", "nvtx_range_start", "nvtx_range_end", + "python_gc_start", + "python_gc_end", "marker" ] }, diff --git a/schemas/host-capture.schema.json b/schemas/host-capture.schema.json index 0fd03bb..5e097d5 100644 --- a/schemas/host-capture.schema.json +++ b/schemas/host-capture.schema.json @@ -396,6 +396,8 @@ "gpu_memset_end", "nvtx_range_start", "nvtx_range_end", + "python_gc_start", + "python_gc_end", "marker" ] }, diff --git a/schemas/measurement-result.schema.json b/schemas/measurement-result.schema.json index 82c3d32..c6e3e7c 100644 --- a/schemas/measurement-result.schema.json +++ b/schemas/measurement-result.schema.json @@ -526,6 +526,8 @@ "gpu_memset_end", "nvtx_range_start", "nvtx_range_end", + "python_gc_start", + "python_gc_end", "marker" ] }, diff --git a/schemas/validate.schema.json b/schemas/validate.schema.json index 60d0783..1a50ed3 100644 --- a/schemas/validate.schema.json +++ b/schemas/validate.schema.json @@ -129,6 +129,8 @@ "gpu_memset_end", "nvtx_range_start", "nvtx_range_end", + "python_gc_start", + "python_gc_end", "marker" ] }, @@ -273,6 +275,12 @@ "ResolvedLinuxSelector": { "type": "object", "properties": { + "binary_path": { + "type": [ + "string", + "null" + ] + }, "category": { "type": "string" }, @@ -285,6 +293,12 @@ "probe_kind": { "$ref": "#/$defs/HostProbeKind" }, + "provider": { + "type": [ + "string", + "null" + ] + }, "syscall_number": { "type": [ "integer", diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index aa34a7e..7368540 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -3,3 +3,7 @@ target_compile_options(xprobe-uprobe-target PRIVATE -O0 -Wall -Wextra -Werror) add_executable(xprobe-syscall-target fixtures/syscall_target.c) target_compile_options(xprobe-syscall-target PRIVATE -O0 -Wall -Wextra -Werror) + +add_executable(xprobe-python-gc-usdt-target fixtures/python_gc_usdt_target.c) +target_compile_options(xprobe-python-gc-usdt-target PRIVATE -O0 -Wall -Wextra -Werror -no-pie) +target_link_options(xprobe-python-gc-usdt-target PRIVATE -no-pie) diff --git a/tests/fixtures/python_gc_usdt_target.c b/tests/fixtures/python_gc_usdt_target.c new file mode 100644 index 0000000..867c601 --- /dev/null +++ b/tests/fixtures/python_gc_usdt_target.c @@ -0,0 +1,38 @@ +#include + +#define XPROBE_USDT_NOTE(provider, name) \ + __asm__ volatile( \ + "990: nop\n" \ + ".pushsection .note.stapsdt,\"?\",@note\n" \ + ".balign 4\n" \ + ".4byte 992f-991f, 994f-993f, 3\n" \ + "991: .asciz \"stapsdt\"\n" \ + "992: .balign 4\n" \ + "993: .8byte 990b\n" \ + ".8byte 0\n" \ + ".8byte 0\n" \ + ".asciz \"" provider "\"\n" \ + ".asciz \"" name "\"\n" \ + ".asciz \"\"\n" \ + "994: .balign 4\n" \ + ".popsection\n") + +static __attribute__((noinline)) void python_gc_start(void) +{ + XPROBE_USDT_NOTE("python", "gc__start"); +} + +static __attribute__((noinline)) void python_gc_done(void) +{ + XPROBE_USDT_NOTE("python", "gc__done"); +} + +int main(void) +{ + for (;;) { + python_gc_start(); + usleep(1000); + python_gc_done(); + usleep(10000); + } +} diff --git a/tests/integration/run-linux-live.sh b/tests/integration/run-linux-live.sh index f645854..846e54b 100755 --- a/tests/integration/run-linux-live.sh +++ b/tests/integration/run-linux-live.sh @@ -2,12 +2,14 @@ set -euo pipefail target=/workspace/build/tests/xprobe-syscall-target +python_gc_target=/workspace/build/tests/xprobe-python-gc-usdt-target binary=/workspace/target/debug/xprobe mmap_capture=/tmp/xprobe-mmap.json munmap_capture=/tmp/xprobe-munmap.json tracepoint_capture=/tmp/xprobe-tracepoint.json capacity_capture=/tmp/xprobe-capacity.json syscall_inventory=/tmp/xprobe-syscall-inventory.json +python_gc_capture=/tmp/xprobe-python-gc.json capacity_stderr=/tmp/xprobe-capacity.stderr "${target}" & @@ -16,10 +18,29 @@ cleanup() { kill "${target_pid}" 2>/dev/null || true wait "${target_pid}" 2>/dev/null || true rm -f "${mmap_capture}" "${munmap_capture}" "${tracepoint_capture}" \ - "${capacity_capture}" "${capacity_stderr}" "${syscall_inventory}" + "${capacity_capture}" "${capacity_stderr}" "${syscall_inventory}" \ + "${python_gc_capture}" } trap cleanup EXIT +"${python_gc_target}" & +python_gc_pid=$! +cleanup_python_gc() { + kill "${python_gc_pid}" 2>/dev/null || true + wait "${python_gc_pid}" 2>/dev/null || true +} +trap 'cleanup_python_gc; cleanup' EXIT + +"${binary}" measure \ + --pid "${python_gc_pid}" \ + --from python:gc_start \ + --to python:gc_end \ + --match exact \ + --samples 3 \ + --max-events 64 \ + --timeout-ms 5000 \ + --json --non-interactive --no-color >"${python_gc_capture}" + "${binary}" measure \ --pid "${target_pid}" \ --syscall-aggregate \ @@ -83,4 +104,6 @@ printf ',"capacity":' cat "${capacity_capture}" printf ',"syscall_inventory":' cat "${syscall_inventory}" +printf ',"python_gc":' +cat "${python_gc_capture}" printf '}\n' diff --git a/tests/integration/test_linux.py b/tests/integration/test_linux.py index 888315f..c598e16 100755 --- a/tests/integration/test_linux.py +++ b/tests/integration/test_linux.py @@ -47,7 +47,8 @@ def main() -> None: assert_tracepoint_capture(captures["tracepoint"]) assert_capacity_failure(captures["capacity"]) assert_syscall_inventory(captures["syscall_inventory"]) - print("captured syscall inventory plus mmap, munmap, and named tracepoint latency evidence") + assert_python_gc_capture(captures["python_gc"]) + print("captured syscall inventory plus syscall, tracepoint, and Python GC latency evidence") def assert_syscall_capture(result: dict, name: str) -> None: @@ -115,5 +116,27 @@ def assert_syscall_inventory(result: dict) -> None: assert mmap["entry_selector_hint"] == "syscall:mmap:entry" +def assert_python_gc_capture(result: dict) -> None: + assert result["ok"] is True + assert result["measurement"]["samples"]["matched"] == 3 + assert result["correlation"]["method"] == "exact_python_gc_tid_lifecycle" + assert result["correlation"]["confidence"] == "exact" + assert result["collection"]["dropped_events"] == 0 + for pair in result["evidence"]: + start = pair["start"] + end = pair["end"] + assert start["event_type"] == "python_gc_start" + assert end["event_type"] == "python_gc_end" + assert start["tid"] == end["tid"] + assert start["host"]["probe_kind"] == "usdt" + assert end["host"]["probe_kind"] == "usdt" + assert start["host"]["symbol"] == "gc__start" + assert end["host"]["symbol"] == "gc__done" + assert start["host"]["arguments"] == [] + assert end["host"]["arguments"] == [] + assert start["attributes"]["usdt_provider"] == "python" + assert pair["latency_ns"] == end["timestamp_ns"] - start["timestamp_ns"] + + if __name__ == "__main__": main() diff --git a/xprobe/collector/src/linux.rs b/xprobe/collector/src/linux.rs index 5fc622c..b6197d9 100644 --- a/xprobe/collector/src/linux.rs +++ b/xprobe/collector/src/linux.rs @@ -81,7 +81,7 @@ impl LinuxCaptureError { ErrorCode::PermissionDenied } Self::Libbpf { - operation: "attach tracepoint", + operation: "attach tracepoint" | "attach USDT", source, } if source.kind() == ErrorKind::NotFound => ErrorCode::InvalidEventSelector, Self::TargetNamespace { .. } @@ -319,7 +319,7 @@ fn collect_raw_events( let ring_buffer = ring_builder .build() .map_err(|source| libbpf_error("build Linux ring buffer", source))?; - let _links = attach_probes(object, &request.probes)?; + let _links = attach_probes(object, request.target.pid, &request.probes)?; arm_object(object)?; if let Some(ready) = &request.ready { ready.send(()).map_err(|_| { @@ -374,6 +374,7 @@ fn arm_object(object: &Object) -> Result<(), LinuxCaptureError> { fn attach_probes( object: &Object, + target_pid: u32, probes: &[ResolvedLinuxSelector], ) -> Result, LinuxCaptureError> { let mut links = Vec::with_capacity(probes.len()); @@ -430,6 +431,28 @@ fn attach_probes( }; links.push(link); } + for (index, probe) in probes.iter().enumerate() { + if probe.probe_kind != HostProbeKind::Usdt { + continue; + } + let program_name = format!("xprobe_handle_usdt_{}", index + 1); + let program = object + .progs_mut() + .find(|program| program.name() == OsStr::new(&program_name)) + .ok_or_else(|| missing("program", &program_name))?; + let pid = i32::try_from(target_pid) + .map_err(|_| LinuxCaptureError::InvalidRequest("target PID exceeds i32".to_owned()))?; + links.push( + program + .attach_usdt( + pid, + probe.binary_path.as_deref().expect("validated USDT path"), + probe.provider.as_deref().expect("validated USDT provider"), + &probe.name, + ) + .map_err(|source| libbpf_error("attach USDT", source))?, + ); + } Ok(links) } @@ -457,9 +480,26 @@ fn validate_request(request: &LinuxCaptureRequest) -> Result<(), LinuxCaptureErr EventType::SyscallEntry | EventType::SyscallExit ) && probe.category == "syscalls" && probe.syscall_number.is_some() + && probe.binary_path.is_none() + && probe.provider.is_none() } HostProbeKind::Tracepoint => { - probe.event_type == EventType::Tracepoint && probe.syscall_number.is_none() + probe.event_type == EventType::Tracepoint + && probe.syscall_number.is_none() + && probe.binary_path.is_none() + && probe.provider.is_none() + } + HostProbeKind::Usdt => { + matches!( + probe.event_type, + EventType::PythonGcStart | EventType::PythonGcEnd + ) && probe.category == "python" + && probe.syscall_number.is_none() + && probe + .binary_path + .as_deref() + .is_some_and(|path| !path.is_empty()) + && probe.provider.as_deref() == Some("python") } _ => false, }; @@ -498,10 +538,15 @@ fn normalize_event( Vec::new() }; let mut attributes = BTreeMap::new(); - attributes.insert( - "tracepoint_category".to_owned(), - Value::String(probe.category.clone()), - ); + if probe.probe_kind == HostProbeKind::Usdt { + let provider = probe.provider.as_ref().expect("validated USDT provider"); + attributes.insert("usdt_provider".to_owned(), Value::String(provider.clone())); + } else { + attributes.insert( + "tracepoint_category".to_owned(), + Value::String(probe.category.clone()), + ); + } Ok(Event { schema_version: SchemaVersion::current(), session_id: session_id.to_owned(), @@ -519,7 +564,7 @@ fn normalize_event( process_start_time: Some(request.target.process_start_time), host: Some(HostEvent { probe_kind: probe.probe_kind.clone(), - binary_path: None, + binary_path: probe.binary_path.clone(), build_id: None, symbol: Some(probe.name.clone()), symbol_demangled: None, @@ -584,6 +629,25 @@ mod tests { category: "syscalls".to_owned(), name: "mmap".to_owned(), syscall_number: Some(9), + binary_path: None, + provider: None, + } + } + + fn python_gc(event_type: EventType) -> ResolvedLinuxSelector { + let name = if event_type == EventType::PythonGcStart { + "gc__start" + } else { + "gc__done" + }; + ResolvedLinuxSelector { + event_type, + probe_kind: HostProbeKind::Usdt, + category: "python".to_owned(), + name: name.to_owned(), + syscall_number: None, + binary_path: Some("/usr/bin/python3".to_owned()), + provider: Some("python".to_owned()), } } @@ -657,6 +721,30 @@ mod tests { assert_eq!(exit.host.unwrap().return_value, Some(-1)); } + #[test] + fn normalizes_python_gc_usdt_without_payloads() { + let request = request(vec![python_gc(EventType::PythonGcStart)]); + let raw = RawEvent { + timestamp_ns: 100, + sequence: 1, + values: [0; 6], + pid: 1234, + tid: 1235, + cpu: 2, + probe_id: 1, + }; + + let event = normalize_event(&raw, &request, "session").unwrap(); + let host = event.host.unwrap(); + assert_eq!(event.event_type, EventType::PythonGcStart); + assert_eq!(host.probe_kind, HostProbeKind::Usdt); + assert_eq!(host.binary_path.as_deref(), Some("/usr/bin/python3")); + assert_eq!(host.symbol.as_deref(), Some("gc__start")); + assert!(host.arguments.is_empty()); + assert_eq!(event.attributes["usdt_provider"], "python"); + assert!(validate_request(&request).is_ok()); + } + #[test] fn rejects_inconsistent_or_unbounded_requests() { let mut invalid = request(vec![ResolvedLinuxSelector { @@ -665,6 +753,8 @@ mod tests { category: "syscalls".to_owned(), name: "mmap".to_owned(), syscall_number: Some(9), + binary_path: None, + provider: None, }]); assert!(matches!( validate_request(&invalid), diff --git a/xprobe/core/src/lib.rs b/xprobe/core/src/lib.rs index 39b4259..e8fb8a7 100644 --- a/xprobe/core/src/lib.rs +++ b/xprobe/core/src/lib.rs @@ -9,4 +9,5 @@ pub mod inject; pub mod inspect; pub mod resolve; pub mod syscall_aggregate; +mod usdt; pub mod validate; diff --git a/xprobe/core/src/usdt.rs b/xprobe/core/src/usdt.rs new file mode 100644 index 0000000..ef49138 --- /dev/null +++ b/xprobe/core/src/usdt.rs @@ -0,0 +1,253 @@ +use std::{ + collections::BTreeSet, + error::Error, + fmt, fs, io, + path::{Path, PathBuf}, +}; + +use object::{Object, ObjectSection}; +use xprobe_protocol::{ErrorCode, ProcessReport}; + +const STAPSDT_SECTION: &str = ".note.stapsdt"; +const STAPSDT_OWNER: &[u8] = b"stapsdt\0"; +const PYTHON_PROVIDER: &str = "python"; +const GC_START: &str = "gc__start"; +const GC_DONE: &str = "gc__done"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PythonGcProbes { + pub binary_path: String, +} + +#[derive(Debug)] +pub enum UsdtError { + Read { path: PathBuf, source: io::Error }, + ParseObject { path: PathBuf, message: String }, + ReadSection { path: PathBuf, message: String }, + MalformedNote { path: PathBuf, message: String }, +} + +impl UsdtError { + #[must_use] + pub fn code(&self) -> ErrorCode { + match self { + Self::Read { source, .. } if source.kind() == io::ErrorKind::PermissionDenied => { + ErrorCode::PermissionDenied + } + Self::Read { .. } + | Self::ParseObject { .. } + | Self::ReadSection { .. } + | Self::MalformedNote { .. } => ErrorCode::Internal, + } + } + + #[must_use] + pub const fn recoverable(&self) -> bool { + matches!(self, Self::Read { .. }) + } +} + +impl fmt::Display for UsdtError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Read { path, source } => { + write!(formatter, "failed to read {}: {source}", path.display()) + } + Self::ParseObject { path, message } => { + write!( + formatter, + "failed to parse {} as an object: {message}", + path.display() + ) + } + Self::ReadSection { path, message } => write!( + formatter, + "failed to read {STAPSDT_SECTION} from {}: {message}", + path.display() + ), + Self::MalformedNote { path, message } => write!( + formatter, + "malformed {STAPSDT_SECTION} in {}: {message}", + path.display() + ), + } + } +} + +impl Error for UsdtError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Read { source, .. } => Some(source), + Self::ParseObject { .. } | Self::ReadSection { .. } | Self::MalformedNote { .. } => { + None + } + } + } +} + +pub fn find_python_gc_probes(report: &ProcessReport) -> Result, UsdtError> { + let mut candidates = BTreeSet::from([PathBuf::from(&report.executable)]); + candidates.extend(report.loaded_libraries.iter().filter_map(|path| { + let candidate = Path::new(path); + candidate + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("libpython") && name.contains(".so")) + .then(|| candidate.to_path_buf()) + })); + + for path in candidates { + let names = stap_probe_names(&path)?; + if names.contains(&(PYTHON_PROVIDER.to_owned(), GC_START.to_owned())) + && names.contains(&(PYTHON_PROVIDER.to_owned(), GC_DONE.to_owned())) + { + return Ok(Some(PythonGcProbes { + binary_path: path.to_string_lossy().into_owned(), + })); + } + } + Ok(None) +} + +fn stap_probe_names(path: &Path) -> Result, UsdtError> { + let bytes = fs::read(path).map_err(|source| UsdtError::Read { + path: path.to_path_buf(), + source, + })?; + let object = object::File::parse(bytes.as_slice()).map_err(|error| UsdtError::ParseObject { + path: path.to_path_buf(), + message: error.to_string(), + })?; + let Some(section) = object.section_by_name(STAPSDT_SECTION) else { + return Ok(BTreeSet::new()); + }; + let data = section.data().map_err(|error| UsdtError::ReadSection { + path: path.to_path_buf(), + message: error.to_string(), + })?; + parse_stapsdt_notes(data, object.is_little_endian(), object.is_64()).map_err(|message| { + UsdtError::MalformedNote { + path: path.to_path_buf(), + message, + } + }) +} + +fn parse_stapsdt_notes( + data: &[u8], + little_endian: bool, + is_64: bool, +) -> Result, String> { + let mut offset = 0_usize; + let mut probes = BTreeSet::new(); + while offset < data.len() { + let namesz = read_u32(data, offset, little_endian)? as usize; + let descsz = read_u32(data, offset + 4, little_endian)? as usize; + offset = offset + .checked_add(12) + .ok_or_else(|| "note header offset overflowed".to_owned())?; + let name_end = offset + .checked_add(namesz) + .ok_or_else(|| "note owner length overflowed".to_owned())?; + let owner = data + .get(offset..name_end) + .ok_or_else(|| "note owner exceeds section bounds".to_owned())?; + offset = align_four(name_end)?; + let desc_end = offset + .checked_add(descsz) + .ok_or_else(|| "note descriptor length overflowed".to_owned())?; + let descriptor = data + .get(offset..desc_end) + .ok_or_else(|| "note descriptor exceeds section bounds".to_owned())?; + offset = align_four(desc_end)?; + if owner == STAPSDT_OWNER { + probes.insert(parse_stapsdt_descriptor(descriptor, is_64)?); + } + } + Ok(probes) +} + +fn parse_stapsdt_descriptor(descriptor: &[u8], is_64: bool) -> Result<(String, String), String> { + let address_bytes = if is_64 { 24 } else { 12 }; + let strings = descriptor + .get(address_bytes..) + .ok_or_else(|| "descriptor does not contain three addresses".to_owned())?; + let (provider, rest) = take_c_string(strings, "provider")?; + let (name, _) = take_c_string(rest, "name")?; + Ok((provider.to_owned(), name.to_owned())) +} + +fn take_c_string<'a>(data: &'a [u8], field: &str) -> Result<(&'a str, &'a [u8]), String> { + let end = data + .iter() + .position(|byte| *byte == 0) + .ok_or_else(|| format!("descriptor {field} is not NUL terminated"))?; + let value = std::str::from_utf8(&data[..end]) + .map_err(|error| format!("descriptor {field} is not UTF-8: {error}"))?; + Ok((value, &data[end + 1..])) +} + +fn read_u32(data: &[u8], offset: usize, little_endian: bool) -> Result { + let end = offset + .checked_add(4) + .ok_or_else(|| "note integer offset overflowed".to_owned())?; + let bytes: [u8; 4] = data + .get(offset..end) + .ok_or_else(|| "note header exceeds section bounds".to_owned())? + .try_into() + .expect("four-byte range"); + Ok(if little_endian { + u32::from_le_bytes(bytes) + } else { + u32::from_be_bytes(bytes) + }) +} + +fn align_four(value: usize) -> Result { + value + .checked_add(3) + .map(|value| value & !3) + .ok_or_else(|| "note alignment overflowed".to_owned()) +} + +#[cfg(test)] +mod tests { + use super::{parse_stapsdt_descriptor, parse_stapsdt_notes}; + + #[test] + fn parses_python_gc_probe_names() { + let descriptor = descriptor("python", "gc__start", "-4@%edi"); + let mut note = Vec::new(); + note.extend_from_slice(&8_u32.to_le_bytes()); + note.extend_from_slice( + &u32::try_from(descriptor.len()) + .expect("test descriptor fits u32") + .to_le_bytes(), + ); + note.extend_from_slice(&3_u32.to_le_bytes()); + note.extend_from_slice(b"stapsdt\0"); + note.extend_from_slice(&descriptor); + while note.len() % 4 != 0 { + note.push(0); + } + let probes = parse_stapsdt_notes(¬e, true, true).unwrap(); + assert!(probes.contains(&("python".to_owned(), "gc__start".to_owned()))); + } + + #[test] + fn rejects_unterminated_descriptor_strings() { + let mut descriptor = vec![0; 24]; + descriptor.extend_from_slice(b"python"); + let error = parse_stapsdt_descriptor(&descriptor, true).unwrap_err(); + assert!(error.contains("provider is not NUL terminated")); + } + + fn descriptor(provider: &str, name: &str, arguments: &str) -> Vec { + let mut descriptor = vec![0; 24]; + for value in [provider, name, arguments] { + descriptor.extend_from_slice(value.as_bytes()); + descriptor.push(0); + } + descriptor + } +} diff --git a/xprobe/core/src/validate.rs b/xprobe/core/src/validate.rs index 79e7285..69c4299 100644 --- a/xprobe/core/src/validate.rs +++ b/xprobe/core/src/validate.rs @@ -12,6 +12,7 @@ use crate::{ cupti_compat, inspect::{self, InspectError}, resolve::{self, ResolveError}, + usdt::{self, UsdtError}, }; #[derive(Debug)] @@ -21,26 +22,29 @@ pub enum ValidateError { endpoint: &'static str, source: ResolveError, }, + Usdt(UsdtError), InvalidSelector(String), InvalidCorrelationPolicy(String), } impl ValidateError { #[must_use] - pub const fn code(&self) -> ErrorCode { + pub fn code(&self) -> ErrorCode { match self { Self::Inspect(error) => error.code(), Self::Resolve { source, .. } => source.code(), + Self::Usdt(error) => error.code(), Self::InvalidSelector(_) => ErrorCode::InvalidEventSelector, Self::InvalidCorrelationPolicy(_) => ErrorCode::InvalidCorrelationPolicy, } } #[must_use] - pub const fn recoverable(&self) -> bool { + pub fn recoverable(&self) -> bool { match self { Self::Inspect(error) => error.recoverable(), Self::Resolve { source, .. } => source.recoverable(), + Self::Usdt(error) => error.recoverable(), Self::InvalidSelector(_) | Self::InvalidCorrelationPolicy(_) => true, } } @@ -53,6 +57,7 @@ impl fmt::Display for ValidateError { Self::Resolve { endpoint, source } => { write!(formatter, "failed to resolve {endpoint} selector: {source}") } + Self::Usdt(error) => error.fmt(formatter), Self::InvalidSelector(reason) => write!(formatter, "invalid event selector: {reason}"), Self::InvalidCorrelationPolicy(policy) => { write!(formatter, "invalid correlation policy {policy:?}") @@ -66,6 +71,7 @@ impl Error for ValidateError { match self { Self::Inspect(error) => Some(error), Self::Resolve { source, .. } => Some(source), + Self::Usdt(error) => Some(error), Self::InvalidSelector(_) | Self::InvalidCorrelationPolicy(_) => None, } } @@ -84,6 +90,8 @@ enum EndpointKind { SyscallEntry { name: String }, SyscallExit { name: String }, Tracepoint, + PythonGcStart, + PythonGcEnd, CudaApi { domain: String, name: String }, Kernel, Memcpy, @@ -108,6 +116,8 @@ impl EndpointKind { | Self::SyscallEntry { .. } | Self::SyscallExit { .. } | Self::Tracepoint + | Self::PythonGcStart + | Self::PythonGcEnd ) } @@ -119,6 +129,8 @@ impl EndpointKind { | Self::SyscallEntry { .. } | Self::SyscallExit { .. } | Self::Tracepoint + | Self::PythonGcStart + | Self::PythonGcEnd | Self::CudaApi { .. } | Self::NvtxRange { .. } ) @@ -313,6 +325,21 @@ fn resolve_endpoint( kind, }); } + if selector.starts_with("python:") { + let (linux, kind, collectable) = parse_python_selector(report, selector)?; + return Ok(Endpoint { + public: ValidatedEndpoint { + selector: selector.to_owned(), + source: EndpointSource::Host, + event_type: linux.event_type.clone(), + collectable, + host: None, + linux: Some(linux), + cuda: None, + }, + kind, + }); + } let (cuda, kind, collectable) = parse_cuda_selector(selector)?; Ok(Endpoint { @@ -371,6 +398,8 @@ fn parse_linux_selector( category: "syscalls".to_owned(), name: (*name).to_owned(), syscall_number: Some(syscall_number), + binary_path: None, + provider: None, }, kind, )) @@ -388,6 +417,8 @@ fn parse_linux_selector( category: (*category).to_owned(), name: (*name).to_owned(), syscall_number: None, + binary_path: None, + provider: None, }, EndpointKind::Tracepoint, )) @@ -396,11 +427,50 @@ fn parse_linux_selector( "tracepoint selector must be tracepoint::".to_owned(), )), _ => Err(ValidateError::InvalidSelector( - "expected uprobe:, syscall:, tracepoint:, or cuda: prefix".to_owned(), + "expected uprobe:, syscall:, tracepoint:, python:, or cuda: prefix".to_owned(), )), } } +fn parse_python_selector( + report: &ProcessReport, + selector: &str, +) -> Result<(ResolvedLinuxSelector, EndpointKind, bool), ValidateError> { + let (event_type, kind, name) = match selector { + "python:gc_start" => ( + EventType::PythonGcStart, + EndpointKind::PythonGcStart, + "gc__start", + ), + "python:gc_end" => ( + EventType::PythonGcEnd, + EndpointKind::PythonGcEnd, + "gc__done", + ), + _ => { + return Err(ValidateError::InvalidSelector( + "Python selector must be python:".to_owned(), + )); + } + }; + let probes = usdt::find_python_gc_probes(report).map_err(ValidateError::Usdt)?; + let binary_path = probes.map(|probes| probes.binary_path); + let collectable = binary_path.is_some(); + Ok(( + ResolvedLinuxSelector { + event_type, + probe_kind: HostProbeKind::Usdt, + category: "python".to_owned(), + name: name.to_owned(), + syscall_number: None, + binary_path, + provider: Some("python".to_owned()), + }, + kind, + collectable, + )) +} + fn syscall_number(name: &str) -> Option { Some(match name { "read" => 0, @@ -779,13 +849,21 @@ fn parse_match_policy(policy: &str) -> Result { fn check_collectability(endpoint: &Endpoint, name: &str, issues: &mut Vec) { if !endpoint.public.collectable { - issues.push(issue( - ErrorCode::InvalidEventSelector, + let message = if matches!( + endpoint.kind, + EndpointKind::PythonGcStart | EndpointKind::PythonGcEnd + ) { + format!( + "{name} selector {} requires mapped CPython ELF metadata exposing python:gc__start and python:gc__done USDT probes", + endpoint.public.selector + ) + } else { format!( "{name} selector {} is recognized but not collected by this build", endpoint.public.selector - ), - )); + ) + }; + issues.push(issue(ErrorCode::InvalidEventSelector, message)); } } @@ -802,6 +880,13 @@ fn check_capabilities( || matches!(end.kind, EndpointKind::HostEntry); let needs_uretprobe = matches!(start.kind, EndpointKind::HostReturn) || matches!(end.kind, EndpointKind::HostReturn); + let needs_usdt = matches!( + start.kind, + EndpointKind::PythonGcStart | EndpointKind::PythonGcEnd + ) || matches!( + end.kind, + EndpointKind::PythonGcStart | EndpointKind::PythonGcEnd + ); let needs_tracepoint = matches!( start.kind, EndpointKind::SyscallEntry { .. } @@ -813,7 +898,7 @@ fn check_capabilities( | EndpointKind::SyscallExit { .. } | EndpointKind::Tracepoint ); - if (needs_uprobe && !report.capabilities.uprobe) + if ((needs_uprobe || needs_usdt) && !report.capabilities.uprobe) || (needs_uretprobe && !report.capabilities.uretprobe) || (needs_tracepoint && !report.capabilities.tracepoint) { @@ -903,7 +988,8 @@ fn supports_exact(start: &EndpointKind, end: &EndpointKind) -> bool { ) => start_domain == end_domain && start_name == end_name, (EndpointKind::Kernel, EndpointKind::Kernel) | (EndpointKind::Memcpy, EndpointKind::Memcpy) - | (EndpointKind::Memset, EndpointKind::Memset) => true, + | (EndpointKind::Memset, EndpointKind::Memset) + | (EndpointKind::PythonGcStart, EndpointKind::PythonGcEnd) => true, ( EndpointKind::NvtxRange { name_regex: start_name, @@ -1083,6 +1169,8 @@ mod tests { category: "syscalls".to_owned(), name: "mmap".to_owned(), syscall_number: Some(9), + binary_path: None, + provider: None, }, ); let end = endpoint( @@ -1097,6 +1185,8 @@ mod tests { category: "syscalls".to_owned(), name: "mmap".to_owned(), syscall_number: Some(9), + binary_path: None, + provider: None, }, ); diff --git a/xprobe/correlator/src/lib.rs b/xprobe/correlator/src/lib.rs index 3b07fd9..f5a5ab1 100644 --- a/xprobe/correlator/src/lib.rs +++ b/xprobe/correlator/src/lib.rs @@ -105,6 +105,10 @@ enum Selector { category: String, name: String, }, + PythonGc { + event_type: EventType, + marker: &'static str, + }, Api { event_type: EventType, domain: String, @@ -145,10 +149,13 @@ impl Selector { if text.starts_with("tracepoint:") { return Self::parse_tracepoint(text); } + if text.starts_with("python:") { + return Self::parse_python(text); + } let fields: Vec<&str> = text.splitn(3, ':').collect(); if fields.first() != Some(&"cuda") || fields.len() < 2 { return Err(MeasureError::InvalidSelector( - "completed captures require a uprobe:, syscall:, tracepoint:, or cuda: selector" + "completed captures require a uprobe:, syscall:, tracepoint:, python:, or cuda: selector" .to_owned(), )); } @@ -286,6 +293,22 @@ impl Selector { }) } + fn parse_python(text: &str) -> Result { + match text { + "python:gc_start" => Ok(Self::PythonGc { + event_type: EventType::PythonGcStart, + marker: "gc__start", + }), + "python:gc_end" => Ok(Self::PythonGc { + event_type: EventType::PythonGcEnd, + marker: "gc__done", + }), + _ => Err(MeasureError::InvalidSelector( + "Python selector must be python:".to_owned(), + )), + } + } + fn parse_kernel(fields: &[&str]) -> Result { let event_type = match fields[1] { "kernel_start" => EventType::GpuKernelStart, @@ -460,6 +483,18 @@ impl Selector { .and_then(serde_json::Value::as_str) == Some(category.as_str()) } + Self::PythonGc { event_type, marker } => { + event.event_type == *event_type + && event.host.as_ref().is_some_and(|host| { + host.probe_kind == HostProbeKind::Usdt + && host.symbol.as_deref() == Some(*marker) + }) + && event + .attributes + .get("usdt_provider") + .and_then(serde_json::Value::as_str) + == Some("python") + } Self::Memcpy { event_type, kind } => { event.event_type == *event_type && kind.as_ref().is_none_or(|kind| { @@ -507,8 +542,30 @@ impl Selector { .. }, ) => start_pattern == end_pattern, - (Self::Host { .. } | Self::Tracepoint { .. } | Self::Nvtx { .. }, _) - | (_, Self::Host { .. } | Self::Tracepoint { .. } | Self::Nvtx { .. }) => false, + ( + Self::PythonGc { + event_type: EventType::PythonGcStart, + .. + }, + Self::PythonGc { + event_type: EventType::PythonGcEnd, + .. + }, + ) => true, + ( + Self::Host { .. } + | Self::Tracepoint { .. } + | Self::Nvtx { .. } + | Self::PythonGc { .. }, + _, + ) + | ( + _, + Self::Host { .. } + | Self::Tracepoint { .. } + | Self::Nvtx { .. } + | Self::PythonGc { .. }, + ) => false, _ => true, } } @@ -547,6 +604,22 @@ impl Selector { ) } + fn is_python_gc_lifecycle(&self, end: &Self) -> bool { + matches!( + (self, end), + ( + Self::PythonGc { + event_type: EventType::PythonGcStart, + .. + }, + Self::PythonGc { + event_type: EventType::PythonGcEnd, + .. + }, + ) + ) + } + fn supports_stack_nested(&self, end: &Self) -> bool { match (self, end) { ( @@ -636,6 +709,13 @@ struct Outcome<'a> { ambiguous: u64, } +#[derive(Debug, Clone, Copy)] +struct LifecycleKinds { + syscall: bool, + nvtx: bool, + python_gc: bool, +} + /// Correlate a bounded event capture and calculate latency statistics. /// /// # Errors @@ -674,15 +754,17 @@ pub fn measure( let clock_domain = common_clock_domain(&starts, &ends)?; apply_duration_window(&mut starts, &mut ends, options.duration)?; - let syscall_lifecycle = start_selector.is_syscall_lifecycle(&end_selector); - let nvtx_lifecycle = start_selector.is_nvtx_lifecycle(&end_selector); + let lifecycles = LifecycleKinds { + syscall: start_selector.is_syscall_lifecycle(&end_selector), + nvtx: start_selector.is_nvtx_lifecycle(&end_selector), + python_gc: start_selector.is_python_gc_lifecycle(&end_selector), + }; let outcome = correlate_selected( &starts, &ends, options.match_policy, options.samples, - syscall_lifecycle, - nvtx_lifecycle, + lifecycles, ); if outcome.pairs.is_empty() { return Err(MeasureError::NoMatchedSamples); @@ -695,8 +777,7 @@ pub fn measure( .map(|pair| pair.latency_ns) .collect::>(); let denominator = matched + outcome.unmatched_start + outcome.unmatched_end + outcome.ambiguous; - let (method, confidence) = - correlation_metadata(options.match_policy, syscall_lifecycle, nvtx_lifecycle); + let (method, confidence) = correlation_metadata(options.match_policy, lifecycles); let warnings = measurement_warnings(options, &starts, &ends); let (host_events, cuda_events) = collection_event_counts(events); @@ -763,12 +844,13 @@ fn correlate_selected<'a>( ends: &[&'a Event], policy: MatchPolicy, limit: Option, - syscall_lifecycle: bool, - nvtx_lifecycle: bool, + lifecycles: LifecycleKinds, ) -> Outcome<'a> { match policy { - MatchPolicy::Exact if syscall_lifecycle => correlate_syscall_lifecycle(starts, ends, limit), - MatchPolicy::Exact if nvtx_lifecycle => correlate_nvtx_lifecycle(starts, ends, limit), + MatchPolicy::Exact if lifecycles.syscall || lifecycles.python_gc => { + correlate_thread_lifecycle(starts, ends, limit) + } + MatchPolicy::Exact if lifecycles.nvtx => correlate_nvtx_lifecycle(starts, ends, limit), MatchPolicy::Exact => correlate_exact(starts, ends, limit), MatchPolicy::FirstAfter => correlate_first_after(starts, ends, limit), MatchPolicy::Nearest => correlate_nearest(starts, ends, limit), @@ -802,16 +884,19 @@ fn validate_policy( const fn correlation_metadata( policy: MatchPolicy, - syscall_lifecycle: bool, - nvtx_lifecycle: bool, + lifecycles: LifecycleKinds, ) -> (&'static str, CorrelationConfidence) { match policy { - MatchPolicy::Exact if syscall_lifecycle => { + MatchPolicy::Exact if lifecycles.syscall => { ("exact_syscall_tid_lifecycle", CorrelationConfidence::Exact) } - MatchPolicy::Exact if nvtx_lifecycle => { + MatchPolicy::Exact if lifecycles.nvtx => { ("exact_nvtx_range_id", CorrelationConfidence::Exact) } + MatchPolicy::Exact if lifecycles.python_gc => ( + "exact_python_gc_tid_lifecycle", + CorrelationConfidence::Exact, + ), MatchPolicy::Exact => ("exact_cupti_correlation_id", CorrelationConfidence::Exact), MatchPolicy::FirstAfter => ("first_after", CorrelationConfidence::Heuristic), MatchPolicy::Nearest => ("nearest", CorrelationConfidence::Heuristic), @@ -820,7 +905,7 @@ const fn correlation_metadata( } } -fn correlate_syscall_lifecycle<'a>( +fn correlate_thread_lifecycle<'a>( starts: &[&'a Event], ends: &[&'a Event], limit: Option, @@ -1546,6 +1631,22 @@ mod tests { event } + fn python_gc_event(event_type: EventType, timestamp: u64, tid: u32) -> Event { + let marker = match event_type { + EventType::PythonGcStart => "gc__start", + EventType::PythonGcEnd => "gc__done", + _ => panic!("Python GC fixture requires a GC boundary"), + }; + let mut event = syscall_event(event_type, timestamp, tid, marker); + event.host.as_mut().unwrap().probe_kind = HostProbeKind::Usdt; + event.host.as_mut().unwrap().binary_path = Some("/usr/bin/python3".to_owned()); + event.attributes.insert( + "usdt_provider".to_owned(), + serde_json::Value::String("python".to_owned()), + ); + event + } + #[test] fn exact_matching_uses_cupti_correlation_ids() { let events = vec![ @@ -1606,6 +1707,34 @@ mod tests { assert_eq!(result.evidence[0].end.tid, 10); } + #[test] + fn exact_python_gc_matching_follows_thread_lifecycle() { + let events = vec![ + python_gc_event(EventType::PythonGcStart, 100, 10), + python_gc_event(EventType::PythonGcStart, 110, 20), + python_gc_event(EventType::PythonGcEnd, 160, 20), + python_gc_event(EventType::PythonGcEnd, 190, 10), + ]; + let options = MeasureOptions { + session_id: "xp_python_gc".to_owned(), + name: Some("python_gc".to_owned()), + start_selector: "python:gc_start".to_owned(), + end_selector: "python:gc_end".to_owned(), + match_policy: MatchPolicy::Exact, + samples: Some(2), + duration: None, + max_events: 100, + dropped_events: 0, + }; + + let result = measure(&events, &options).unwrap(); + assert_eq!(result.measurement.samples.matched, 2); + assert_eq!(result.measurement.latency_ns.min, 50); + assert_eq!(result.measurement.latency_ns.max, 90); + assert_eq!(result.correlation.method, "exact_python_gc_tid_lifecycle"); + assert_eq!(result.correlation.confidence, CorrelationConfidence::Exact); + } + #[test] fn exact_nvtx_matching_uses_kind_and_range_id() { let events = vec![ diff --git a/xprobe/exporter/src/lib.rs b/xprobe/exporter/src/lib.rs index 69555da..fa4f63b 100644 --- a/xprobe/exporter/src/lib.rs +++ b/xprobe/exporter/src/lib.rs @@ -107,6 +107,8 @@ const fn event_type_name(event_type: &EventType) -> &'static str { EventType::GpuMemsetEnd => "gpu_memset_end", EventType::NvtxRangeStart => "nvtx_range_start", EventType::NvtxRangeEnd => "nvtx_range_end", + EventType::PythonGcStart => "python_gc_start", + EventType::PythonGcEnd => "python_gc_end", EventType::Marker => "marker", } } diff --git a/xprobe/protocol/src/event.rs b/xprobe/protocol/src/event.rs index acdc9d9..eee9de6 100644 --- a/xprobe/protocol/src/event.rs +++ b/xprobe/protocol/src/event.rs @@ -35,6 +35,8 @@ pub enum EventType { GpuMemsetEnd, NvtxRangeStart, NvtxRangeEnd, + PythonGcStart, + PythonGcEnd, Marker, } diff --git a/xprobe/protocol/src/validate.rs b/xprobe/protocol/src/validate.rs index e8c21d1..157d4d4 100644 --- a/xprobe/protocol/src/validate.rs +++ b/xprobe/protocol/src/validate.rs @@ -42,6 +42,10 @@ pub struct ResolvedLinuxSelector { pub category: String, pub name: String, pub syscall_number: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub binary_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] diff --git a/xprobe/protocol/tests/contracts.rs b/xprobe/protocol/tests/contracts.rs index 3e20e90..790c90d 100644 --- a/xprobe/protocol/tests/contracts.rs +++ b/xprobe/protocol/tests/contracts.rs @@ -5,9 +5,9 @@ use serde_json::{Value, json}; use xprobe_protocol::{ AggregateInventoryResult, CapabilityReport, CpuSampleInventoryResult, CpuSamplingSpec, CpuSamplingValidationResult, DiscoveryResult, ErrorResponse, Event, HostCaptureResult, - MeasurementResult, MeasurementSpec, ProcessReport, ResolvedProbe, SyscallAggregateResult, - SyscallAggregateSpec, SyscallAggregateValidationResult, TraceExportResult, ValidationResult, - schema::generated_schemas, + MeasurementResult, MeasurementSpec, ProcessReport, ResolvedLinuxSelector, ResolvedProbe, + SyscallAggregateResult, SyscallAggregateSpec, SyscallAggregateValidationResult, + TraceExportResult, ValidationResult, schema::generated_schemas, }; fn assert_round_trip(fixture: &Value) @@ -521,6 +521,19 @@ fn resolved_probe_contract_round_trips() { })); } +#[test] +fn resolved_python_usdt_contract_round_trips() { + assert_round_trip::(&json!({ + "event_type": "python_gc_start", + "probe_kind": "usdt", + "category": "python", + "name": "gc__start", + "syscall_number": null, + "binary_path": "/usr/bin/python3.12", + "provider": "python" + })); +} + #[test] fn validation_result_contract_round_trips() { assert_round_trip::(&json!({ From e201e136fd86a53038bc91fe927d5849bfaf6877 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 1 Aug 2026 05:12:14 +0800 Subject: [PATCH 06/15] =?UTF-8?q?=F0=9F=A4=96=20feat:=20guide=20CPU=20and?= =?UTF-8?q?=20Python=20investigations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AGENTS.md | 6 + README.md | 45 ++--- docs/agent-integration.md | 23 ++- docs/architecture.md | 45 ++++- docs/cli-contract.md | 75 +++++++- docs/development.md | 15 +- justfile | 5 + skills/xprobe-measure-latency/SKILL.md | 161 +++++++++--------- .../xprobe-measure-latency/agents/openai.yaml | 2 +- .../examples/cpu-hotspot-inventory.json | 16 ++ .../examples/python-gc-duration.json | 15 ++ .../examples/syscall-inventory.json | 12 ++ .../references/cli-contract.md | 47 ++++- .../references/investigation.md | 101 ++++++++--- .../references/multi-process.md | 12 +- .../references/result-quality.md | 42 +++-- .../fixtures/workflow-routes.json | 34 ++++ tests/agent-contract/test_contract.py | 61 +++++-- tests/agent-contract/test_workflow_routes.py | 41 +++++ xprobe/cli/src/main.rs | 2 +- 20 files changed, 597 insertions(+), 163 deletions(-) create mode 100644 skills/xprobe-measure-latency/examples/cpu-hotspot-inventory.json create mode 100644 skills/xprobe-measure-latency/examples/python-gc-duration.json create mode 100644 skills/xprobe-measure-latency/examples/syscall-inventory.json create mode 100644 tests/agent-contract/fixtures/workflow-routes.json create mode 100644 tests/agent-contract/test_workflow_routes.py diff --git a/AGENTS.md b/AGENTS.md index 0acc444..e42cc4e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,6 +22,9 @@ - Resolve architecture-specific syscall names in core. In BPF, filter process identity and syscall number before reading scalar registers or reserving a record. Arm multi-link collectors only after every link is attached. +- Resolve runtime probes from mapped binary capabilities, not process names. + libbpf USDT attachment requires its support maps even when xprobe does not + read marker arguments. ## Failures and safety @@ -43,6 +46,9 @@ default. Named tracepoints retain identity and timestamps unless a versioned scalar payload is explicitly designed. Never describe temporal correlation as exact causality. +- Treat sampled hotspot proportions as estimates. Preserve sample loss, stack + truncation, thread coverage, symbol coverage, and native fallback when Python + semantic symbols are unavailable. ## Agent workflow diff --git a/README.md b/README.md index 84baecb..9be70b4 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,10 @@ `xprobe` is an AI harness for measuring latency between two observable events in a process, on the CPU, NVIDIA GPU, or across both. Its bounded native -profiler combines eBPF function, syscall, and tracepoint evidence with NVIDIA -CUPTI, an agent-friendly CLI, strict JSON contracts, explicit correlation -quality, and no daemon or server lifecycle. +profiler combines sampled native/Python stacks, eBPF function, syscall, +tracepoint and CPython runtime evidence with NVIDIA CUPTI, an agent-friendly +CLI, strict JSON contracts, explicit correlation quality, and no daemon or +server lifecycle. ## Install @@ -35,25 +36,26 @@ Node.js is only needed for Skill installation, not for xprobe itself. See ## Measure -Confirm the local capabilities first. `ok: true` means diagnosis completed; -read the individual checks before selecting events. +For an unknown CPU workload, start with one representative sampled inventory: ```bash -xprobe doctor --json --non-interactive --no-color +xprobe validate --pid 4242 --cpu-sample \ + --json --non-interactive --no-color -xprobe discover --pid 4242 --limit 50 \ +xprobe measure --pid 4242 \ + --cpu-sample --duration-ms 1000 \ --json --non-interactive --no-color +``` + +Use its hotspot and selector evidence, or a GPU aggregate inventory, to state a +narrow hypothesis. Then validate and measure that boundary directly: +```bash xprobe validate --pid 4242 \ --from 'cuda:runtime_api:cudaLaunchKernel:exit' \ --to 'cuda:kernel_start:name~flash.*' \ --match exact --json --non-interactive --no-color -xprobe measure --pid 4242 \ - --from 'cuda:kernel_start' --to 'cuda:kernel_end' \ - --match exact --aggregate --duration-ms 1000 --max-groups 4096 \ - --json --non-interactive --no-color - xprobe measure --pid 4242 \ --from 'cuda:runtime_api:cudaLaunchKernel:exit' \ --to 'cuda:kernel_start:name~flash.*' \ @@ -62,12 +64,11 @@ xprobe measure --pid 4242 \ --json --non-interactive --no-color ``` -Kernel launch latency is only one event pair. The same workflow measures host -function spans, syscall latency, named Linux events, CUDA API calls, GPU -operation durations, transfers, NVTX application ranges, and paths across CPU -and GPU events after selecting the correct process. Aggregate mode provides a -bounded coarse inventory of GPU operations before an exact evidence -measurement narrows the question. +Kernel launch latency is only one event pair. xprobe can first inventory sampled +CPU/Python hotspots, syscall lifecycles, or GPU activity, then measure host +function spans, CPython GC, syscall latency, named Linux events, CUDA API calls, +GPU operation durations, transfers, NVTX ranges, and CPU/GPU paths after +selecting the correct process. `measure` also accepts completed `--input` captures and versioned live `--spec` files. Evidence can be exported as `jsonl` or `chrome`. JSON results @@ -82,8 +83,8 @@ is also preserved when correlation or clock validation fails. | --- | --- | | `doctor` | Report local eBPF, ptrace, NVIDIA, CUDA, and CUPTI capabilities | | `discover` | List NVML-confirmed CUDA context holders under a process-tree root | -| `validate` | Resolve two selectors and report collection, mutation, clock, and policy requirements without attaching | -| `measure` | Collect or import bounded events, correlate pairs, emit statistics and full event evidence | +| `validate` | Check an event pair or inventory mode and report requirements without attaching | +| `measure` | Inventory bounded work or correlate bounded event evidence | `measure --pid` automatically loads the matching CUDA 12 or CUDA 13 CUPTI Agent when a selected endpoint requires it. It reports the target mutation on stderr @@ -97,7 +98,9 @@ attach cannot retrofit an initialized NVTX dispatch. | Surface | Current support | | --- | --- | | OS/architecture | Linux x86_64, glibc 2.34 or newer | -| Host events | PID-scoped ELF function, named syscall, and tracepoint boundaries | +| CPU inventory | PID-scoped sampled user stacks with native ELF and CPython perf-map symbols | +| Host events | ELF functions, named syscall/tracepoint boundaries, and CPython GC USDT | +| Coarse aggregation | Bounded syscall lifecycle and CUDA kernel/memcpy/memset groups | | CUDA callbacks | Runtime and Driver API entry/exit | | GPU activity | Kernel, memcpy, and memset start/end | | Application ranges | Bounded ASCII NVTX thread and process ranges | diff --git a/docs/agent-integration.md b/docs/agent-integration.md index af595f2..4f1ad45 100644 --- a/docs/agent-integration.md +++ b/docs/agent-integration.md @@ -27,10 +27,14 @@ install its whole directory so its references, examples, and analysis script remain available. The Skill routes setup, completed-artifact analysis, known-boundary measurement, -unknown CPU/GPU investigation, and multi-process work independently. It checks -or installs the CLI only for live work, uses broad inventory only when selectors -are unknown, and includes `scripts/analyze_trace.py` for deterministic kernel, -copy, overlap, stream, and gap summaries. The xprobe repository tests +unknown CPU/Python/GPU investigation, and multi-process work independently. It +checks or installs the CLI only for live work. Unknown CPU work starts with +sampled stacks, adds syscall aggregation or CPython GC only when supported by a +hypothesis, and falls back visibly to native frames when Python semantics are +unavailable. Unknown GPU work uses only relevant aggregates. Mixed CPU/GPU +inventories may run concurrently with separate contracts, bounds, outputs, and +failure handling. The bundled `scripts/analyze_trace.py` provides deterministic +kernel, copy, overlap, stream, and gap summaries. The repository tests installation with `skills` CLI 1.5.20 in isolated home directories. This pinned test protects released behavior while the documented `skills@1` selector receives compatible path updates. @@ -42,6 +46,12 @@ measurement per selected worker concurrently. Results, warnings, failures, and artifacts remain per process; xprobe does not add a multi-process command or claim cross-process causality. +Inventory outputs are not event artifacts. CPU hotspots, syscall groups, and +GPU groups produce selector hypotheses; every exact selector still passes +read-only `validate`. The Agent must inspect the quality fields specific to each +schema and cannot equate sample proportions, aggregate duration shares, or +overlapping capture windows with exact causality. + ## Contract test ```bash @@ -52,8 +62,9 @@ just test-skill-install The test requires the visible command set to be exactly `doctor`, `discover`, `validate`, and `measure`. It invokes the first three in strict JSON mode, checks injection requirements, verifies schemas, exercises the bundled trace -analyzer, and checks adaptive task routing, bounded live collection, mutation -guards, and result quality/evidence. +analyzer, and checks adaptive task routing for unknown CPU, Python, mixed, +known-selector, existing-artifact, and unsupported-runtime scenarios, bounded +live collection, mutation guards, and result quality/evidence. The installation test uses the real third-party CLI in isolated home directories and verifies byte-for-byte copies for Codex, Claude Code, and Cursor. diff --git a/docs/architecture.md b/docs/architecture.md index cb63ad0..9ceb5df 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -33,7 +33,7 @@ attachment, collection, correlation, logical CUPTI shutdown, and cleanup. | `xprobe/collector` | eBPF collection, CUPTI control protocol and ABI decoding | | `xprobe/correlator` | Selector matching, pair evidence, statistics and quality | | `xprobe/exporter` | Event JSONL and Chrome Trace Event Format | -| `bpf/` | PID-scoped uprobe, syscall, and named tracepoint programs | +| `bpf/` | PID-scoped uprobe, USDT, syscall, tracepoint, and aggregate programs | | `cupti/` | Reusable in-process CUDA callback/activity agent | ## Identity and discovery @@ -58,9 +58,17 @@ symbol table. A full C++ signature uses the explicit `symbol=` selector form; the resolver checks exported dynamic symbols first, falls back to the complete table, and returns both the attachable mangled name and readable signature. +CPU stack analysis resolves sampled instruction pointers through the target's +mapped ELF load segments and symbol tables. CPython frames are resolved +independently from `/tmp/perf-.map` when the interpreter exposes an active +perf trampoline. Native coverage remains usable when Python semantic symbols +are inactive or unsupported. CPython GC resolution parses `.note.stapsdt` from +the mapped interpreter or `libpython` and accepts the selector only when both +required markers are present in one ELF. + ## Validation -`validate` is read-only. It resolves ELF probes and Linux syscall numbers, +`validate` is read-only. It resolves ELF, USDT, and Linux syscall probes, parses named tracepoint and CUDA selectors, checks correlation-policy compatibility, and reports eBPF, CUPTI, callback, activity, and clock requirements. CUPTI activation is one of: @@ -78,6 +86,11 @@ table, so validation returns an explicit issue instead of promising injection. Malformed selectors, unresolved symbols, invalid policies, and unavailable required host collection remain explicit issues or errors. +CPU sampling and syscall aggregation use dedicated validation contracts. They +check target identity and current perf-event or eBPF capability without +attaching, report no target mutation, and preserve unavailable Python semantic +symbolization as status rather than inventing frames. + ## Collection Host events use libbpf-rs. ELF functions attach one PID-scoped uprobe or @@ -95,6 +108,25 @@ layout. Ring exhaustion, scalar-read failure, malformed records, and duration-capacity exhaustion remain explicit failures. Rust ownership detaches all links on every return path. +CPython GC uses the same bounded Linux ring and two `SEC("usdt")` programs. +libbpf attaches each marker to the exact mapped ELF returned by validation; +both links attach before the shared map is armed. Events retain identity, +timestamp, provider, and marker name only. Exact correlation follows one +ordered start/end lifecycle per process thread. + +CPU inventory opens one bounded software CPU-clock perf event per current +target thread and samples userspace callchains into fixed capacities. It counts +exact raw stacks before deterministic top-group truncation, then resolves native +ELF and optional Python perf-map frames into stack groups and +inclusive/exclusive hotspots. Sample loss, truncated callchains, thread fanout, +group pressure, and symbol coverage remain first-class output. + +Syscall inventory uses PID-filtered raw entry/exit programs with a bounded LRU +map for per-thread starts and a bounded per-CPU hash for aggregate groups. The +hot path retains syscall number, timestamp, scalar return status, and integer +duration accounting; it emits no per-call ring records. Userspace merges the +per-CPU values and maps known x86_64 numbers to selector hints. + CUDA events use an in-process CUPTI Agent. CUDA 12 and CUDA 13 builds share the same source and capture ABI but link their matching CUPTI SONAME. Loading the Agent creates a control socket; `measure` separately arms one fresh, @@ -112,7 +144,12 @@ TID with the returned nesting level; process keys use the returned 64-bit NVTX range ID and permit a different end thread. STOP disables callback domains but does not unsubscribe the routing required by CUDA 13. -Broad GPU inventory uses the same `measure` primitive with aggregate mode. The +Broad inventory remains in the foreground `measure` primitive. CPU sampling, +syscall aggregation, and GPU aggregation have separate result schemas because +their quality and capacity meanings differ. Their groups are hypotheses for a +later validated exact capture, not event timelines that can be re-correlated. + +Broad GPU inventory uses aggregate mode. The Agent updates a `--max-groups`-bounded table for matching kernel, memcpy, or memset activity and returns only final count/duration/byte summaries. Exact measurement remains the evidence path; aggregate output has a separate result @@ -156,7 +193,7 @@ All sources normalize into the versioned `Event` type. `measure` supports: | Policy | Pairing | Confidence | | --- | --- | --- | -| `exact` | CUPTI correlation ID, NVTX range ID, or same-thread syscall lifecycle | exact | +| `exact` | CUPTI correlation ID, NVTX range ID, or same-thread syscall/CPython GC lifecycle | exact | | `first-after` | First unused end at or after each start | heuristic | | `nearest` | Nearest unused end by timestamp | heuristic | | `stack-nested` | Per-thread LIFO host entry/return | high | diff --git a/docs/cli-contract.md b/docs/cli-contract.md index e5c95d0..351a758 100644 --- a/docs/cli-contract.md +++ b/docs/cli-contract.md @@ -71,6 +71,8 @@ uprobe::+0x:return syscall::entry syscall::exit tracepoint:: +python:gc_start +python:gc_end ``` The `symbol=` form allows `::` and other punctuation in a full C++ signature. @@ -86,6 +88,12 @@ return value on exit. It never dereferences pointer arguments. Named tracepoints record identity and timestamp only; they do not copy the tracepoint payload. Unsupported syscall names and unavailable tracepoints fail explicitly. +Python GC selectors are capability-based, not process-name based. Validation +reads the target's mapped CPython executable and `libpython` ELF metadata and +requires both `python:gc__start` and `python:gc__done` USDT markers. Captured +events contain process/thread identity, timestamp, mapped binary, provider, and +marker name without copying generation or object payloads. + CUDA forms include Runtime/Driver API entry/exit, kernel/memcpy/memset activity start/end, and NVTX range boundaries. Kernel selectors accept `name~REGEX`; memcpy selectors accept `kind=`. NVTX selectors are @@ -102,13 +110,17 @@ xprobe validate --pid 4242 \ --match exact --json --non-interactive --no-color ``` -Validation is read-only. It verifies target identity, resolves host selectors, -parses CUDA filters, and checks collection and correlation requirements. +Validation is read-only. It verifies target identity, resolves host and Python +USDT selectors, parses CUDA filters, and checks collection and correlation +requirements. `validate --cpu-sample` checks target threads, perf-event access, +and Python symbolization status. `validate --syscall-aggregate` checks the +bounded eBPF aggregate path. Inventory modes do not take event-pair selectors. Results conform to `schemas/validate.schema.json`. Supported policies are `exact`, `first-after`, `nearest`, `stack-nested`, and `stream-order`. Exact uses deterministic CUPTI correlation IDs, NVTX range -kind/ID, or one named syscall's per-thread entry/exit lifecycle. Nested +kind/ID, one named syscall's per-thread entry/exit lifecycle, or one CPython GC +start/end lifecycle on the same thread. Nested requires entry/return of the same host function. Stream order requires GPU activity endpoints. Temporal policies always warn that they are heuristic. `policy_recommendation` reports the strongest compatible policy, a stable @@ -152,6 +164,32 @@ xprobe measure --pid 4242 \ --json --non-interactive --no-color ``` +CPU/Python hotspot inventory: + +```bash +xprobe measure --pid 4242 --cpu-sample --duration-ms 1000 \ + --frequency-hz 99 --max-samples 10000 --max-groups 256 \ + --stack-depth 64 --max-threads 1024 \ + --json --non-interactive --no-color +``` + +Syscall lifecycle inventory: + +```bash +xprobe measure --pid 4242 --syscall-aggregate --duration-ms 1000 \ + --max-groups 256 --max-inflight 1024 \ + --json --non-interactive --no-color +``` + +CPython garbage-collection duration: + +```bash +xprobe measure --pid 4242 \ + --from python:gc_start --to python:gc_end \ + --match exact --samples 100 --max-events 1000 \ + --json --non-interactive --no-color +``` + NVTX application range: ```bash @@ -191,6 +229,28 @@ xprobe measure --pid 4242 \ Exactly one source mode is used: `--pid`, one or more `--input`, or `--spec`. At least one positive `--samples` or `--duration-ms` bound is required in direct mode. `--timeout-ms` defaults to 30 seconds and `--max-events` to 100,000. + +`--cpu-sample` is live-only and requires `--duration-ms`. It opens bounded +userspace callchain sampling events for the target's current threads. Defaults +are 99 Hz, 10,000 observed samples, 256 retained stack groups, depth 64, and +1,024 threads. The result uses +`schemas/cpu-sample-inventory-result.schema.json`: exact raw stack counts are +grouped deterministically, hotspots report inclusive/exclusive counts and +proportions, and attachable native frames include uprobe selector hints. Native +ELF and `/tmp/perf-.map` Python symbols are resolved independently. +`python_status` and resolved/unresolved frame counts prevent native fallback +from being mislabeled as Python semantic coverage. Lost samples, truncated +stacks, thread coverage, and capacity pressure remain visible. + +`--syscall-aggregate` is live-only and requires `--duration-ms`. PID-filtered +raw syscall entry/exit programs retain at most `--max-inflight` thread starts +and `--max-groups` syscall groups in BPF maps, then emit one bounded summary. +Known x86_64 numbers include names and exact entry/exit selector hints; unknown +numbers remain numeric. `schemas/syscall-aggregate-result.schema.json` exposes +entries, matched and unmatched exits, inflight-at-end, drops, occupancy, errors, +and total/min/max/mean duration. It contains no pointer values or per-call event +stream. + `--aggregate` is live-only, duration-bounded, and accepts one matching kernel, memcpy, or memset activity start/end pair. It uses `--max-groups` (default 4,096), does not accept `--samples` or `--events-out`, and emits @@ -204,9 +264,16 @@ Each kernel group reports `name_complete`. CUPTI names that fill the fixed prefix instead of claiming an exact full name, so the next capture can still filter in the Agent hot path. +The three inventory modes have distinct result contracts and contain no exact +event timeline. They cannot be passed to `measure --input` or exported with +`--events-out`. Their groups and hotspots are evidence for choosing selectors; +read-only validation still precedes the resulting exact measurement. + Live host endpoints attach PID-scoped eBPF probes. Linux syscall endpoints use raw tracepoints so they do not depend on tracingfs event IDs; ordinary named -tracepoints use their kernel category and name. A samples-bound Linux capture +tracepoints use their kernel category and name. CPython GC endpoints attach to +the exact mapped ELF selected by validation. Every endpoint link is attached +before collection is armed. A samples-bound Linux capture allows bounded startup slack for a target already inside an event boundary. A duration capture that fills `--max-events` returns `EVENT_RATE_TOO_HIGH` rather than reporting partial success. CUDA endpoints automatically activate the CUPTI diff --git a/docs/development.md b/docs/development.md index b42d717..489b710 100644 --- a/docs/development.md +++ b/docs/development.md @@ -63,12 +63,25 @@ just test-bpf-live ``` The live suite captures function entry/return, mmap/munmap lifecycle, generic -raw tracepoint, and host-capacity failure paths from controlled targets. It +raw tracepoint, CPython-compatible GC USDT lifecycle, syscall aggregate, and +host-capacity failure paths from controlled targets. It requires Docker daemon access and grants the container `BPF`, `PERFMON`, `SYS_ADMIN`, and `SYS_RESOURCE`, with seccomp disabled for BPF/perf syscalls. It does not use `--privileged`, does not require GPU access, mounts the workspace read-only, and removes the container after the test. +Run the ignored live CPU sampling tests on a host whose perf policy permits +sibling-process sampling: + +```bash +just test-cpu-live +``` + +They cover a busy native process and CPython 3.12+ `-X perf` symbolization. +The target interpreter must provide perf trampoline support for the Python +case; unsupported interpreters remain a valid native-only product path but are +not a substitute for this live gate. + Resolve real CPython, native extension, and libtorch C++ symbols with a local Python environment containing PyTorch: diff --git a/justfile b/justfile index eaa145d..99350a3 100644 --- a/justfile +++ b/justfile @@ -21,11 +21,13 @@ test: build python3 tests/agent-contract/test_contract.py target/debug/xprobe python3 tests/agent-contract/test_trace_analysis.py python3 tests/agent-contract/test_multi_process_workflow.py + python3 tests/agent-contract/test_workflow_routes.py test-agent-contract: build python3 tests/agent-contract/test_contract.py target/debug/xprobe python3 tests/agent-contract/test_trace_analysis.py python3 tests/agent-contract/test_multi_process_workflow.py + python3 tests/agent-contract/test_workflow_routes.py test-skill-install: tests/agent-contract/test_skill_install.sh @@ -44,6 +46,9 @@ test-bpf-live: build python3 tests/integration/test_uprobe.py "{{cuda_smoke_image}}" python3 tests/integration/test_linux.py "{{cuda_smoke_image}}" +test-cpu-live: build + cargo test -p xprobe-cli --test cpu_sampling -- --ignored --test-threads=1 + test-cupti: cmake -S . -B build -G Ninja -DXPROBE_BUILD_CUPTI=ON cmake --build build --target xprobe-cupti-smoke diff --git a/skills/xprobe-measure-latency/SKILL.md b/skills/xprobe-measure-latency/SKILL.md index 7d37fc1..4894d50 100644 --- a/skills/xprobe-measure-latency/SKILL.md +++ b/skills/xprobe-measure-latency/SKILL.md @@ -1,104 +1,109 @@ --- name: xprobe-measure-latency -description: Profile live or recorded Linux CPU and NVIDIA CUDA workloads with bounded xprobe evidence. Use when an agent needs to install or repair xprobe, inspect an existing xprobe JSONL artifact, inventory unknown CPU/GPU activity, validate and measure a known function, syscall, CUDA API, kernel, transfer, synchronization point, NVTX range, or event-to-event latency, compare selected worker processes, investigate a performance regression, or decide when to hand an isolated kernel or host span to another profiler. +description: Profile live or recorded Linux CPU, CPython, and NVIDIA CUDA workloads with bounded xprobe evidence. Use when an agent needs to install or repair xprobe, inventory unknown CPU/GPU work, find native or Python hotspots, inspect syscall or CPython GC cost, validate and measure a known function, syscall, CUDA API, kernel, transfer, synchronization point, NVTX range, or event-to-event latency, compare workers, investigate regressions, or inspect an existing xprobe artifact. --- # Profile workloads with xprobe -Choose the shortest route that answers the user's question. Do not run the full -investigation playbook when the target, selectors, or completed artifact already -provide the missing evidence. +Choose the shortest route supported by the question and existing evidence. Do +not run a fixed checklist or collect every source. ## Route the task - **Existing artifact**: Read [references/trace-analysis.md](references/trace-analysis.md) - and [references/result-quality.md](references/result-quality.md). Analyze the - artifact directly or use `measure --input` to test compatible selectors or a - policy. Skip installation, `doctor`, `discover`, and live attachment unless a - separate live capture is actually needed. + and [references/result-quality.md](references/result-quality.md). Analyze it + directly or use `measure --input`; skip installation, `doctor`, `discover`, + and live attachment unless a new capture is required. - **Known live boundary**: Read [references/cli-contract.md](references/cli-contract.md) - and [references/result-quality.md](references/result-quality.md). Confirm the - target identity, validate the supplied selectors, and run a bounded measure. - Do not run a broad inventory solely to satisfy a checklist. -- **Unknown CPU workload**: Read - [references/investigation.md](references/investigation.md), classify the - suspected host boundary, and narrow from application, symbol, syscall, or - tracepoint evidence. Do not run CUDA discovery. + and [references/result-quality.md](references/result-quality.md). Confirm PID + plus start time, run read-only `validate`, then one bounded measurement. Do + not run a broad inventory solely to satisfy a checklist. +- **Unknown CPU or Python workload**: Read + [references/investigation.md](references/investigation.md). Start with bounded + `--cpu-sample` evidence. Add `--syscall-aggregate` only for a kernel-facing + hypothesis; validate CPython GC boundaries only for a GC hypothesis. Narrow + from emitted selector hints rather than guessing or requiring manual symbol + inspection first. Do not run CUDA discovery. - **Unknown GPU or mixed workload**: Read - [references/investigation.md](references/investigation.md). Establish - readiness, discover CUDA context holders, collect only the broad bounded - inventories needed by the question, derive selectors from evidence, then - validate and measure narrowly. + [references/investigation.md](references/investigation.md). Use `discover` + only to select CUDA context holders. Collect the relevant bounded GPU + aggregate and, for mixed work, a bounded CPU sample inventory. Run independent + coarse captures concurrently when an aligned workload window matters, with + separate bounds, outputs, failures, and perturbation accounting. - **Multiple processes**: Also read - [references/multi-process.md](references/multi-process.md). Select relevant - PID/start-time identities and run independent bounded commands concurrently - when aligned capture windows matter. Keep every result and artifact separate. + [references/multi-process.md](references/multi-process.md). Select explicit + PID/start-time identities. Inventory one representative per defensible worker + class, then run independent narrow commands concurrently where useful. - **Setup or repair**: Read [references/setup.md](references/setup.md) only when - live commands are needed and the CLI is absent, incompatible, or unhealthy. - A completed-artifact analysis does not require a local collector. - -For live work, classify the selected path as CPU-only or GPU/mixed before -choosing collectors. Run `doctor` when capability is unknown or a command -reports an environment failure; it is not a prerequisite for every valid -offline or already-diagnosed workflow. + a required live command is absent, incompatible, or unhealthy. Existing + artifacts do not require a local collector. + +Classify live work as CPU-only or GPU/mixed before choosing collectors. Run +`doctor` when capability is unknown or a command reports an environment +failure, not as a prerequisite for valid offline or already-diagnosed work. + +## Narrow from evidence + +For unknown CPU work, validate and run one representative `--cpu-sample` +window. Inspect sample loss, stack truncation, thread coverage, symbol coverage, +`python_status`, stack groups, and inclusive/exclusive hotspots. An active +Python perf map provides Python frames; `inactive` or `unsupported` does not +invalidate native frames. Use native or Python hotspot context and emitted +entry/return selector hints to form the next hypothesis. + +Use `--syscall-aggregate` when CPU evidence or the question points to I/O, +allocation, scheduling, or VM behavior. Inspect unmatched/inflight/drop and map +capacity before using a group's selector hints. For suspected CPython garbage +collection, validate `python:gc_start` to `python:gc_end`; validation checks the +target's ELF USDT metadata and may reject a build without those probes. + +For unknown GPU work, aggregate only relevant kernel, memcpy, or memset +families over a representative cycle. Read group counts, duration totals, +bytes, completeness, and selector hints. Scope breadth and capture duration are +independent: narrowing scope is not a substitute for retaining a representative +window. + +Every hotspot or aggregate group is a hypothesis, not final attribution. Pass +the selected exact endpoints through read-only `validate`, honor its policy +recommendation explicitly, and collect one detailed bounded measurement per +stated hypothesis. A failed validation revises the hypothesis; it does not +justify unrelated collection. ## Preserve these invariants - Use JSON mode and require schema version `2.0`. Treat malformed output and - unknown schema versions as errors. -- Identify each live target by PID plus procfs start time. Recheck the identity - around validation and attachment; never substitute a newly observed PID. -- Run read-only `validate` before every live measurement or target mutation. - Use its explicit policy recommendation, but never change policy silently. -- Bound every capture by samples or duration, timeout, and exact-event or - aggregate-group capacity. Scope breadth and capture duration are independent. -- When validation reports `injection_required`, disclose that `measure` will - ptrace the process and leave the CUPTI shared object mapped. When it reports - `startup_required` for NVTX, restart with the matching Agent before the first - NVTX call; online injection cannot retrofit initialized NVTX dispatch. -- Inspect status, collection completeness, buffer utilization, unmatched, - ambiguous, and dropped counts, clock alignment and estimated error, - correlation method, confidence, and every evidence pair before interpreting - a result. -- Keep stream and process identity in every claim. Summed concurrent GPU + unknown versions as errors. +- Identify a live process by PID plus procfs start time and verify it around + validation and attachment. Never substitute a reused PID. +- Bound every capture by duration or samples, timeout, and record/group/thread + capacity. Preserve each command's stdout, stderr, status, and artifact. +- Before CUDA injection, disclose that `measure` will ptrace the process and + leave the CUPTI shared object mapped. `startup_required` NVTX work must restart + with the matching Agent before the first NVTX call. +- Inspect completeness, loss/drops, capacities, unmatched/ambiguous evidence, + symbol and stack coverage, clock alignment, method, confidence, and evidence + pairs before interpreting latency. +- Keep process and CUDA stream identity in every claim. Summed concurrent GPU duration is not wall time; temporal correlation is not exact causality. -## Choose collection depth from evidence - -Use broad-to-narrow collection when selectors are unknown. Keep the broad scope -representative but collect aggregate kernel, memcpy, or memset families only -when they could answer the current question. Use selector hints, counts, -duration totals, transfer bytes, and bounds to state a narrower hypothesis. - -When a trustworthy selector is supplied by the user, application, an NVTX -range, a previous artifact, or a matching build's symbol inspection, validate -it directly. A failed validation is evidence to revise the selector; it is not -a reason to run unrelated inventories. - -Use one bounded live capture per stated hypothesis when source activation or -capture windows differ. A single exact Event JSONL artifact may support several -offline correlations without reattaching. Independent worker captures or -non-conflicting hypotheses may run concurrently when the caller can preserve -their bounds, outputs, failures, and perturbation separately. - -Record an application baseline when the question concerns a regression, -slowdown, or profiler overhead. Warm up readiness-sensitive framework/JIT work -before selecting a representative window. Do not require a new baseline for -schema validation or a purely offline artifact question. +Record a baseline when investigating regression or profiler overhead. Warm up +framework and JIT work before a representative capture. Independent CPU and GPU +inventories can run concurrently, but compare their perturbation and never +merge their distinct result contracts as if they were one timeline. For exact GPU artifacts, run `scripts/analyze_trace.py` and inspect launch -variants, stream distribution, `busy_union_ns`, overlap factor, and adjacent -gaps. Aggregate output has no event ordering and cannot be re-correlated. +variants, stream distribution, `busy_union_ns`, overlap factor, and gaps. +Aggregate and sampled inventories contain no event timeline and cannot be +re-correlated offline. ## Stop or hand off deliberately Stop on target reuse, permission failure, invalid selectors, unavailable -required collectors, drops, incomplete capture, or a clock/correlation problem -that invalidates the intended claim. Preserve failed-capture artifacts and -structured details instead of reporting partial success. A quality limitation -that does not affect the requested same-domain claim may be reported explicitly -rather than treated as a universal stop condition. - -Stop using xprobe once evidence isolates unexplained time inside one kernel; -use NCU or PC sampling for microarchitectural behavior. Use a CPU sampling -profiler when the unresolved time is inside an uninstrumented host span. +required collectors, incomplete collection, loss/drops, or a clock/correlation +problem that invalidates the claim. Preserve structured failures and artifacts. + +Once evidence isolates unexplained time inside one GPU kernel, use NCU or PC +sampling for microarchitecture. Once native sampling isolates code without an +observable boundary, use a runtime-specific profiler or instrumentation. Do not +claim xprobe identifies Python semantics when `python_status` or symbolization +coverage says otherwise. diff --git a/skills/xprobe-measure-latency/agents/openai.yaml b/skills/xprobe-measure-latency/agents/openai.yaml index db83b67..e5d9b8b 100644 --- a/skills/xprobe-measure-latency/agents/openai.yaml +++ b/skills/xprobe-measure-latency/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Xprobe Workload Profiling" - short_description: "Route bounded CPU and GPU profiling tasks" + short_description: "Route bounded CPU, Python, and GPU profiling" default_prompt: "Use $xprobe-measure-latency to choose the shortest evidence-based route for this profiling task." diff --git a/skills/xprobe-measure-latency/examples/cpu-hotspot-inventory.json b/skills/xprobe-measure-latency/examples/cpu-hotspot-inventory.json new file mode 100644 index 0000000..f7110b4 --- /dev/null +++ b/skills/xprobe-measure-latency/examples/cpu-hotspot-inventory.json @@ -0,0 +1,16 @@ +{ + "schema_version": "2.0", + "name": "cpu_hotspot_inventory", + "target": { + "pid": 1234, + "process_start_time": 987654 + }, + "sample_event": "cpu_clock", + "frequency_hz": 99, + "duration_ms": 1000, + "timeout_ms": 30000, + "max_samples": 10000, + "max_groups": 256, + "stack_depth": 64, + "max_threads": 1024 +} diff --git a/skills/xprobe-measure-latency/examples/python-gc-duration.json b/skills/xprobe-measure-latency/examples/python-gc-duration.json new file mode 100644 index 0000000..cdab62b --- /dev/null +++ b/skills/xprobe-measure-latency/examples/python-gc-duration.json @@ -0,0 +1,15 @@ +{ + "schema_version": "2.0", + "name": "python_gc_duration", + "target": { + "pid": 1234, + "process_start_time": 987654 + }, + "start_selector": "python:gc_start", + "end_selector": "python:gc_end", + "match_policy": "exact", + "samples": 100, + "duration_ms": null, + "timeout_ms": 30000, + "max_events": 1000 +} diff --git a/skills/xprobe-measure-latency/examples/syscall-inventory.json b/skills/xprobe-measure-latency/examples/syscall-inventory.json new file mode 100644 index 0000000..61dff2f --- /dev/null +++ b/skills/xprobe-measure-latency/examples/syscall-inventory.json @@ -0,0 +1,12 @@ +{ + "schema_version": "2.0", + "name": "syscall_inventory", + "target": { + "pid": 1234, + "process_start_time": 987654 + }, + "duration_ms": 1000, + "timeout_ms": 30000, + "max_groups": 256, + "max_inflight": 1024 +} diff --git a/skills/xprobe-measure-latency/references/cli-contract.md b/skills/xprobe-measure-latency/references/cli-contract.md index 577e26d..248c8f7 100644 --- a/skills/xprobe-measure-latency/references/cli-contract.md +++ b/skills/xprobe-measure-latency/references/cli-contract.md @@ -28,7 +28,9 @@ readable signature. This resolves native code mapped by a Python process, not Python frame or `module.qualname` names. Linux selectors use `syscall::entry|exit` and -`tracepoint::`. Syscall entry/exit for one name supports +`tracepoint::`. CPython GC selectors use +`python:gc_start|gc_end`; `validate` resolves the mapped executable or +`libpython` ELF containing both required USDT markers. Syscall entry/exit for one name supports `exact` per-thread lifecycle matching. Entry evidence contains scalar ABI register values and exit evidence contains the scalar return value; xprobe does not dereference pointers. Named tracepoints contain no payload fields. @@ -46,7 +48,8 @@ complete selector and policy before measurement. ## Correlation Use `exact` for CUDA events with the same CUPTI correlation ID, NVTX boundaries -with the same range kind and ID, or entry/exit of one named syscall. +with the same range kind and ID, entry/exit of one named syscall, or CPython GC +start/end on one thread. Use `stack-nested` for entry/return of the same host function and `stream-order` for activity events on one CUDA stream. `first-after` and `nearest` are temporal heuristics and cannot establish causality. Read `policy_recommendation.policy`, @@ -59,6 +62,40 @@ Direct `measure` calls require a positive `--samples` or `--duration-ms` bound. `--timeout-ms` defaults to 30 seconds and `--max-events` to 100,000. Use exactly one source mode: `--pid`, one or more `--input` files, or `--spec`. +CPU sampling is live and duration-bounded: + +```bash +xprobe validate --pid "$PID" --cpu-sample \ + --json --non-interactive --no-color +xprobe measure --pid "$PID" --cpu-sample --duration-ms 1000 \ + --frequency-hz 99 --max-samples 10000 --max-groups 256 \ + --stack-depth 64 --max-threads 1024 \ + --json --non-interactive --no-color +``` + +It samples userspace callchains with `perf_event_open` for a bounded snapshot of +the target's current threads. The result reports exact raw stack counts before +bounded grouping, inclusive/exclusive hotspots, attachable native selector +hints, native/Python/unresolved symbol coverage, Python perf-map status, lost +samples, truncated stacks, attached threads, and every capacity. It does not +emit Event JSONL or claim a continuous all-thread trace. Hotspot proportions +have sampling uncertainty and are not exact CPU-time shares. + +Syscall inventory is also live and duration-bounded: + +```bash +xprobe validate --pid "$PID" --syscall-aggregate \ + --json --non-interactive --no-color +xprobe measure --pid "$PID" --syscall-aggregate --duration-ms 1000 \ + --max-groups 256 --max-inflight 1024 \ + --json --non-interactive --no-color +``` + +PID-filtered raw tracepoints retain per-thread starts and per-syscall aggregate +count/error/duration state in bounded BPF maps. The result reports map quality, +inflight/unmatched/drop counts, known names, and entry/exit selector hints. It +does not stream one event per syscall or dereference arguments. + Use `measure --aggregate --duration-ms ... --max-groups ...` for a live, coarse kernel, memcpy, or memset inventory. Aggregate endpoints must be one matching activity start/end pair with `--match exact`. The result contains bounded group @@ -79,11 +116,15 @@ sets a live stop from ARM completion; either samples or duration may complete a call when both are present. Timeout bounds the complete foreground operation and cleanup. -Linux syscall filtering runs in BPF before event reservation. A duration-bound +Linux exact syscall filtering runs in BPF before event reservation. A duration-bound host capture that reaches `max-events` fails with `EVENT_RATE_TOO_HIGH`; narrow the selector or increase the explicit bound. Do not replace it with partial evidence. +CPU sampling, syscall aggregation, and GPU aggregation have distinct schemas. +Treat their hotspots/groups as selector hypotheses; they are not completed +event captures and cannot be passed to `measure --input`. + `--events-out PATH` atomically writes the bounded capture with mode `0600`, not only matched evidence. Collection completeness and CUPTI capacity, observed, retained, dropped, and buffer utilization fields describe capture integrity diff --git a/skills/xprobe-measure-latency/references/investigation.md b/skills/xprobe-measure-latency/references/investigation.md index b38443b..e548142 100644 --- a/skills/xprobe-measure-latency/references/investigation.md +++ b/skills/xprobe-measure-latency/references/investigation.md @@ -18,10 +18,39 @@ use PID plus procfs start time; never reuse an old PID-only choice. ## Map broadly before selecting -When kernel names are unknown, first validate and collect all kernel activity. Choose `REPRESENTATIVE_WINDOW_MS` to cover one steady-state request, batch, or -iteration cycle, not an arbitrarily short interval. The capture is always -bounded; its duration must still preserve the behavior being diagnosed. +iteration cycle, not an arbitrarily short interval. Select the broad source +from the question: sampled stacks for unknown CPU time, syscall aggregation for +kernel-facing behavior, and activity aggregation for unknown GPU operations. +Do not run all three by default. + +Unknown CPU or Python work starts with sampling: + +```bash +xprobe validate --pid "$PID" --cpu-sample \ + --json --non-interactive --no-color + +xprobe measure --pid "$PID" --cpu-sample \ + --duration-ms "$REPRESENTATIVE_WINDOW_MS" \ + --frequency-hz 99 --max-samples 10000 --max-groups 256 \ + --stack-depth 64 --max-threads 1024 --timeout-ms "$TIMEOUT_MS" \ + --json --non-interactive --no-color > coarse-cpu.json +``` + +Use syscall aggregation only when the question or sampled stacks suggest I/O, +memory mapping, allocation, scheduling, or another kernel-facing boundary: + +```bash +xprobe validate --pid "$PID" --syscall-aggregate \ + --json --non-interactive --no-color + +xprobe measure --pid "$PID" --syscall-aggregate \ + --duration-ms "$REPRESENTATIVE_WINDOW_MS" \ + --max-groups 256 --max-inflight 1024 --timeout-ms "$TIMEOUT_MS" \ + --json --non-interactive --no-color > coarse-syscalls.json +``` + +When kernel names are unknown, collect bounded aggregate kernel activity: ```bash xprobe validate --pid "$PID" \ @@ -58,7 +87,14 @@ measurement; device-specific groups may still need workload-level GPU routing. Treat group capacity as a consequence of workload diversity, not event rate. On `EVENT_RATE_TOO_HIGH`, split event families or reduce selector scope while retaining a representative cycle; reduce duration only when the remaining -window is still representative. Aggregate mode never returns partial output. +window is still representative. CPU, syscall, and GPU aggregate modes never +turn incomplete output into successful evidence. + +For a mixed workload whose CPU and GPU phases must cover the same controlled +request, launch the CPU sample and relevant GPU aggregate as independent +concurrent commands. Use distinct output paths and bounds, preserve either +failure, and measure workload throughput against a no-profiler baseline. These +inventories support narrowing but do not establish cross-source causality. ## Derive CUDA selectors @@ -79,8 +115,19 @@ contains literal and validate it before collection. ## Derive CPU selectors -Choose the narrowest observable boundary supported by existing evidence. For a -function, resolve the mapped object in the target and inspect its symbols: +Read `collection.completeness`, `observed_samples`, `grouped_samples`, +`lost_samples`, stack truncation, thread coverage, and table utilization before +ranking `inventory.hotspots`. +Then read resolved native/Python/unresolved frame totals and `python_status`. +An `active` Python map can expose Python function frames. An `inactive` or +`unsupported` status uses a visible native fallback; incomplete Python coverage +also leaves native CPython, extension, framework, and system-library frames +usable. Report that fallback instead of claiming Python semantic attribution. + +Prefer each hotspot's `entry_selector_hint` and `return_selector_hint`. These +are emitted only for attachable native ELF symbols observed in this target and +still require validation. When no hint exists and an exact native boundary is +necessary, the agent may inspect the mapped object itself: ```bash readlink -f "/proc/$PID/exe" @@ -99,12 +146,12 @@ local code, derive a file offset with `readelf`/`objdump` and use `validate`; do not infer a runtime virtual address from one process and reuse it as a file offset. -For eager PyTorch, inspect the mapped CPython executable, `torch._C`, and the -loaded libtorch objects. Prefer an exported dispatcher or native operator -signature observed in that exact installed build, such as an -`at::_ops::::call(...)` boundary, and validate both entry and return -before measuring with `stack-nested`. Treat `_PyEval_EvalFrameDefault` only as a -broad interpreter boundary; xprobe does not turn it into Python function names. +For eager PyTorch, prefer a sampled hotspot or emitted selector hint from the +mapped CPython executable, `torch._C`, or a loaded libtorch object. A validated +exported dispatcher or native operator signature such as an +`at::_ops::::call(...)` boundary can be measured with `stack-nested`. +Treat `_PyEval_EvalFrameDefault` only as a native interpreter boundary when +Python semantic frames are unavailable. After `torch.compile` or Triton warmup, do not assume an eager operator boundary still encloses the fused work. Inventory CUDA kernels first and narrow using @@ -113,13 +160,27 @@ symbol is not a valid uprobe target. Never reuse a C++ signature, mangled name, file offset, or generated kernel name across PyTorch builds without resolving and validating it again. -For kernel-facing latency, first use application logs, `/proc` state, or a -bounded syscall summary to identify a candidate. Then validate +For kernel-facing latency, use application evidence or a bounded syscall +summary to identify a candidate. Then validate `syscall:NAME:entry` to `syscall:NAME:exit` with `exact`. Use `tracepoint:CATEGORY:NAME` only when the kernel event itself is the intended boundary. Do not start an unknown high-rate workload with unfiltered raw syscall tracepoints: select first, then collect detailed evidence. +For a garbage-collection hypothesis, check target capability rather than its +process name: + +```bash +xprobe validate --pid "$PID" \ + --from python:gc_start --to python:gc_end --match exact \ + --json --non-interactive --no-color +``` + +If valid, measure a bounded number of exact same-thread GC lifecycles. If the +mapped CPython build lacks `python:gc__start` and `python:gc__done` USDT notes, +validation rejects the selector; retain sampled native/Python evidence and do +not substitute a guessed symbol. + ## Measure one narrow hypothesis Choose one next boundary from evidence: @@ -129,6 +190,7 @@ Choose one next boundary from evidence: - kernel end to next activity start with `stream-order` for one-stream gaps; - host function entry to return with `stack-nested` for CPU span; - named syscall entry to exit with `exact` for kernel-facing latency; +- Python GC start to end with `exact` for one CPython collection lifecycle; - host marker to GPU activity with `first-after` only as a disclosed heuristic. After capture, analyze the artifact and all result quality fields. An aggregate @@ -150,8 +212,9 @@ skills/xprobe-measure-latency/scripts/analyze_trace.py selected-kernel.jsonl \ ## Escalate at the right boundary -xprobe can isolate slow kernels, launch gaps, copies, synchronization boundaries, -host spans, and host-to-GPU timing. Once the remaining time is inside a single -kernel, use NCU or PC sampling for stalls, cache behavior, occupancy, instruction -mix, or Tensor Core utilization. Use a CPU sampling profiler when the unresolved -time is inside an uninstrumented host span. +xprobe can isolate sampled CPU hotspots, CPython GC, syscall cost, slow kernels, +launch gaps, copies, synchronization boundaries, host spans, and host-to-GPU +timing. Once the remaining time is inside a single kernel, use NCU or PC sampling +for stalls, cache behavior, occupancy, instruction mix, or Tensor Core +utilization. When sampled native code has no stable observable boundary, hand it +to a runtime-specific profiler or add application instrumentation. diff --git a/skills/xprobe-measure-latency/references/multi-process.md b/skills/xprobe-measure-latency/references/multi-process.md index 30ad2e7..fc44702 100644 --- a/skills/xprobe-measure-latency/references/multi-process.md +++ b/skills/xprobe-measure-latency/references/multi-process.md @@ -13,7 +13,8 @@ merely because it owns a CUDA context. Treat workers as homogeneous only when application configuration and observed workload evidence support that claim. For a homogeneous rank set, use one -representative worker for each broad aggregate inventory. Derive narrow +representative worker for each required CPU sample, syscall, or GPU aggregate +inventory. Derive narrow selectors from that inventory, then apply those selectors to every selected worker. For heterogeneous workers, inventory one representative per defensible class or investigate workers separately. @@ -51,12 +52,21 @@ so their capture windows cover the same controlled request, batch, or iteration cycle. Do not serialize the commands unless the investigation explicitly needs different workload windows. +The same rule applies to mixed evidence on one worker: a CPU sample inventory +and GPU aggregate may run as two concurrent xprobe commands when they must cover +the same controlled window. They keep independent duration, timeout, sample or +group capacities, result schemas, stderr, and output paths. Do not merge them or +infer exact CPU/GPU causality from overlapping command times. Avoid concurrent +broad CUPTI captures for the same process because Agent activation and shared +workload perturbation make them poor independent observations. + Keep these outputs separate for every worker: - spec with the discovered target identity; - validation JSON and stderr; - measurement JSON and stderr; - exact Event JSONL artifact when requested; +- separate CPU, syscall, and GPU inventory JSON when collected; - command exit status and start/end wall timestamps. Wait for every command. Do not cancel sibling commands or discard successful diff --git a/skills/xprobe-measure-latency/references/result-quality.md b/skills/xprobe-measure-latency/references/result-quality.md index 1b13f22..daa9656 100644 --- a/skills/xprobe-measure-latency/references/result-quality.md +++ b/skills/xprobe-measure-latency/references/result-quality.md @@ -2,7 +2,8 @@ Prefer `exact` when CUDA endpoints carry the same CUPTI correlation ID or when one named syscall has entry/exit records from the same thread and process -identity. Prefer `stack-nested` for entry/return pairs of the same host +identity. CPython GC exact matching likewise requires ordered start/end markers +on the same thread and process identity. Prefer `stack-nested` for entry/return pairs of the same host function. Use `stream-order` only for GPU activity endpoints on the same device, context, and stream. Treat `first-after` and `nearest` as temporal heuristics, never request causality. @@ -14,11 +15,24 @@ changed, collection was incomplete, or clock alignment failed. Report unmatched and ambiguous counts with matched samples. `estimated_error_ns: null` means no quantified interpolation error bound. -Aggregate inventory is a different contract. Require `completeness: complete`, -zero dropped activities, equal observed and grouped activity counts, and -reasonable table utilization before using its group names or selector hints. -Its min/max/mean come from exact integer totals, but it has no event ordering, -percentiles, stream overlap, or correlation evidence. +Inventory modes are separate contracts: + +- For CPU sampling, require `completeness: complete`, zero lost samples, + `observed_samples == grouped_samples`, acceptable stack truncation, expected + thread coverage, and reasonable group-table utilization. Read native, + Python, and unresolved frame counts before using hotspot names. Sample + proportions have sampling uncertainty and are not exact time shares. +- For syscall aggregation, require `completeness: complete`, zero dropped + aggregates, acceptable unmatched/inflight counts, and reasonable group-table + utilization. Durations are exact for retained matched lifecycles, but groups + contain no event order or percentiles. +- For GPU aggregate inventory, require `completeness: complete`, zero dropped + activities, equal observed and grouped activity counts, and reasonable table + utilization. Min/max/mean derive from integer totals, but there is no event + ordering, stream overlap, or correlation evidence. + +Do not compare capacities or completeness fields across these schemas as if +they described the same collector. ## Concurrency @@ -55,10 +69,12 @@ minimum_records = samples * (start_records_per_sample + end_records_per_sample) max_events >= minimum_records + expected_unmatched_records ``` -Use at least 2x headroom for stable narrow selectors. For broad activity -inventory, use aggregate mode and size `max-groups` from expected operation -diversity rather than event rate. Keep enough duration to cover a representative -cycle; table saturation is an explicit failure. +Use at least 2x headroom for stable narrow selectors. For broad aggregate +inventory, size `max-groups` from expected operation or syscall diversity rather +than event rate. For CPU sampling, size `max-samples` from frequency times +duration, `max-threads` from target fanout, `stack-depth` from expected call +depth, and `max-groups` from stack diversity. Keep enough duration to cover a +representative cycle; saturation or sample loss is an explicit quality failure. `duration-ms` limits correlation to a window beginning at the first selected event. In live mode it also sets a collection stop from ARM completion, so finish @@ -71,5 +87,7 @@ cleanup limit. Record an application-level latency distribution before profiling and repeat it afterward under the same workload. Report the difference and whether automatic injection occurred. The injected shared object remains mapped but is logically -disabled after collection. Do not interpret a one-off profiled request as an -unperturbed baseline. +disabled after collection. For mixed concurrent inventories, report each +collector's wall time, available CPU/RSS observations, and combined workload +throughput. Do not interpret a one-off profiled request as an unperturbed +baseline or claim universal superiority over another profiler. diff --git a/tests/agent-contract/fixtures/workflow-routes.json b/tests/agent-contract/fixtures/workflow-routes.json new file mode 100644 index 0000000..f9788d5 --- /dev/null +++ b/tests/agent-contract/fixtures/workflow-routes.json @@ -0,0 +1,34 @@ +{ + "scenarios": [ + { + "name": "existing_artifact", + "sources": ["skill"], + "required": ["measure --input", "skip installation", "unless a new capture is required"] + }, + { + "name": "known_selector", + "sources": ["skill"], + "required": ["known live boundary", "run read-only validate", "do not run a broad inventory solely to satisfy a checklist"] + }, + { + "name": "unknown_cpu", + "sources": ["skill", "investigation"], + "required": ["unknown cpu or python workload", "--cpu-sample", "observed_samples", "entry_selector_hint"] + }, + { + "name": "python_semantics", + "sources": ["skill", "investigation"], + "required": ["python_status", "native frames", "python:gc_start", "capability rather than its process name"] + }, + { + "name": "mixed_cpu_gpu", + "sources": ["skill", "investigation"], + "required": ["bounded cpu sample inventory", "gpu aggregate", "independent concurrent commands", "separate bounds, outputs, failures"] + }, + { + "name": "unsupported_python_runtime", + "sources": ["skill", "investigation"], + "required": ["`inactive` or `unsupported`", "native fallback", "do not claim xprobe identifies python semantics"] + } + ] +} diff --git a/tests/agent-contract/test_contract.py b/tests/agent-contract/test_contract.py index 83b42cb..9e5d350 100755 --- a/tests/agent-contract/test_contract.py +++ b/tests/agent-contract/test_contract.py @@ -53,7 +53,7 @@ def check_skill(workspace: pathlib.Path) -> None: for route in ( "Existing artifact", "Known live boundary", - "Unknown CPU workload", + "Unknown CPU or Python workload", "Unknown GPU or mixed workload", "Multiple processes", "Setup or repair", @@ -61,18 +61,20 @@ def check_skill(workspace: pathlib.Path) -> None: assert route in normalized_skill for adaptive_rule in ( "Choose the shortest route", - "Skip installation, `doctor`, `discover`, and live attachment", + "skip installation, `doctor`, `discover`, and live attachment", "Do not run a broad inventory solely to satisfy a checklist", "Do not run CUDA discovery", - "collect only the broad bounded inventories needed by the question", - "A completed-artifact analysis does not require a local collector", + "Start with bounded `--cpu-sample` evidence", + "Add `--syscall-aggregate` only for a kernel-facing hypothesis", + "inventories can run concurrently", + "Existing artifacts do not require a local collector", "Run `doctor` when capability is unknown", ): assert adaptive_rule in normalized_skill for invariant in ( "schema version `2.0`", "PID plus procfs start time", - "Run read-only `validate` before every live measurement", + "Pass the selected exact endpoints through read-only `validate`", "Bound every capture", "leave the CUPTI shared object mapped", "temporal correlation is not exact causality", @@ -81,11 +83,11 @@ def check_skill(workspace: pathlib.Path) -> None: for quality_field in ( "unmatched", "ambiguous", - "dropped", - "collection completeness", - "buffer utilization", + "loss/drops", + "completeness", + "symbol and stack coverage", "clock alignment", - "correlation method", + "method", "confidence", "evidence pair", ): @@ -94,6 +96,8 @@ def check_skill(workspace: pathlib.Path) -> None: "CPU-only", "GPU or mixed", "Scope breadth and capture duration are independent", + "python:gc_start", + "native frames", "scripts/analyze_trace.py", "selector hints", "busy_union_ns", @@ -120,6 +124,18 @@ def check_skill(workspace: pathlib.Path) -> None: for example in examples: specification = json.loads(example.read_text()) assert specification["schema_version"] == "2.0", example + if "sample_event" in specification: + modes.add("cpu_sample") + assert specification["frequency_hz"] > 0, example + assert specification["max_samples"] > 0, example + assert specification["stack_depth"] > 0, example + assert specification["max_threads"] > 0, example + continue + if "max_inflight" in specification and "start_selector" not in specification: + modes.add("syscall_aggregate") + assert specification["max_groups"] > 0, example + assert specification["max_inflight"] > 0, example + continue mode = specification.get("measurement_mode", "exact") modes.add(mode) if mode == "aggregate": @@ -128,12 +144,15 @@ def check_skill(workspace: pathlib.Path) -> None: else: assert specification["max_events"] > 0, example policies.add(specification["match_policy"]) - assert modes == {"exact", "aggregate"} + assert modes == {"exact", "aggregate", "cpu_sample", "syscall_aggregate"} assert {"exact", "first_after", "stack_nested", "stream_order"} <= policies openai_yaml = (skill_root / "agents/openai.yaml").read_text() assert 'display_name: "Xprobe Workload Profiling"' in openai_yaml - assert 'short_description: "Route bounded CPU and GPU profiling tasks"' in openai_yaml + assert ( + 'short_description: "Route bounded CPU, Python, and GPU profiling"' + in openai_yaml + ) assert "$xprobe-measure-latency" in openai_yaml investigation = (skill_root / "references/investigation.md").read_text() @@ -146,6 +165,9 @@ def check_skill(workspace: pathlib.Path) -> None: normalized_multi_process = re.sub(r"\s+", " ", multi_process) normalized_trace_analysis = re.sub(r"\s+", " ", trace_analysis) normalized_setup = re.sub(r"\s+", " ", setup) + normalized_cli_contract = re.sub( + r"\s+", " ", (skill_root / "references/cli-contract.md").read_text() + ) for required in ( "Triton", "procfs start time", @@ -154,6 +176,10 @@ def check_skill(workspace: pathlib.Path) -> None: "NO_MATCHED_SAMPLES", "representative cycle", "concrete Runtime or Driver API name", + "observed_samples", + "python_status", + "python:gc_start", + "independent concurrent commands", ): assert required in normalized_investigation for required in ( @@ -161,8 +187,10 @@ def check_skill(workspace: pathlib.Path) -> None: "first selected event", "ARM completion", "Summed kernel", - "Aggregate inventory", + "Inventory modes are separate contracts", "max-groups", + "lost samples", + "stack truncation", ): assert required in normalized_quality for required in ( @@ -174,6 +202,7 @@ def check_skill(workspace: pathlib.Path) -> None: "Never reuse an artifact path", "Do not concatenate artifacts", "overall experiment incomplete", + "CPU sample inventory and GPU aggregate", ): assert required in normalized_multi_process for required in ( @@ -184,6 +213,14 @@ def check_skill(workspace: pathlib.Path) -> None: "distinct capture windows", ): assert required in normalized_trace_analysis + for required in ( + "python:gc_start|gc_end", + "--cpu-sample", + "--syscall-aggregate", + "sampling uncertainty", + "cannot be passed to `measure --input`", + ): + assert required in normalized_cli_contract for required in ( "v0.4.1/install.sh", "xprobe `0.4.x`", diff --git a/tests/agent-contract/test_workflow_routes.py b/tests/agent-contract/test_workflow_routes.py new file mode 100644 index 0000000..12569b4 --- /dev/null +++ b/tests/agent-contract/test_workflow_routes.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +import json +import pathlib +import re + + +def normalize(text: str) -> str: + return re.sub(r"\s+", " ", text.replace("`", "")).lower() + + +def main() -> None: + workspace = pathlib.Path(__file__).resolve().parents[2] + skill_root = workspace / "skills/xprobe-measure-latency" + sources = { + "skill": normalize((skill_root / "SKILL.md").read_text()), + "investigation": normalize( + (skill_root / "references/investigation.md").read_text() + ), + } + fixture = json.loads( + (workspace / "tests/agent-contract/fixtures/workflow-routes.json").read_text() + ) + covered = set() + for scenario in fixture["scenarios"]: + document = " ".join(sources[name] for name in scenario["sources"]) + for phrase in scenario["required"]: + assert normalize(phrase) in document, (scenario["name"], phrase) + covered.add(scenario["name"]) + assert covered == { + "existing_artifact", + "known_selector", + "unknown_cpu", + "python_semantics", + "mixed_cpu_gpu", + "unsupported_python_runtime", + } + print(json.dumps({"schema_version": "2.0", "ok": True, "routes": sorted(covered)})) + + +if __name__ == "__main__": + main() diff --git a/xprobe/cli/src/main.rs b/xprobe/cli/src/main.rs index 8304f29..d1df995 100644 --- a/xprobe/cli/src/main.rs +++ b/xprobe/cli/src/main.rs @@ -283,7 +283,7 @@ struct MeasureArgs { #[arg(long)] agent: Option, - /// Write matched start/end evidence events to this file. + /// Write the complete bounded event capture to this file. #[arg(long)] events_out: Option, From 6e02f9f8083dd6c34544b8d8a17f5a38de1824a4 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 1 Aug 2026 05:19:33 +0800 Subject: [PATCH 07/15] =?UTF-8?q?=F0=9F=93=8A=20perf:=20benchmark=20CPU=20?= =?UTF-8?q?inventory=20perturbation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- benchmarks/cpu-inventory/native_workload.c | 100 ++++ benchmarks/cpu-inventory/python_workload.py | 56 ++ benchmarks/cpu-inventory/run.py | 600 ++++++++++++++++++++ justfile | 3 + 4 files changed, 759 insertions(+) create mode 100644 benchmarks/cpu-inventory/native_workload.c create mode 100644 benchmarks/cpu-inventory/python_workload.py create mode 100644 benchmarks/cpu-inventory/run.py diff --git a/benchmarks/cpu-inventory/native_workload.c b/benchmarks/cpu-inventory/native_workload.c new file mode 100644 index 0000000..9d780e3 --- /dev/null +++ b/benchmarks/cpu-inventory/native_workload.c @@ -0,0 +1,100 @@ +#define _POSIX_C_SOURCE 200809L + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static volatile sig_atomic_t running = 1; +static volatile uint64_t sink; + +static void stop_workload(int signal_number) +{ + (void)signal_number; + running = 0; +} + +static uint64_t monotonic_ns(void) +{ + struct timespec value; + + if (clock_gettime(CLOCK_MONOTONIC, &value) != 0) { + perror("clock_gettime"); + exit(2); + } + return (uint64_t)value.tv_sec * 1000000000ULL + (uint64_t)value.tv_nsec; +} + +__attribute__((noinline)) uint64_t xprobe_native_hot_loop(uint64_t seed) +{ + uint64_t value = seed | 1U; + + for (uint64_t index = 0; index < 500000U; ++index) { + value ^= value << 13U; + value ^= value >> 7U; + value ^= value << 17U; + } + return value; +} + +static void write_metrics(const char *path, uint64_t iterations) +{ + char replacement[4096]; + FILE *output; + + if (snprintf(replacement, sizeof(replacement), "%s.next", path) < 0 || + strlen(path) + sizeof(".next") > sizeof(replacement)) { + fprintf(stderr, "metrics path is too long\n"); + exit(3); + } + output = fopen(replacement, "w"); + if (output == NULL) { + perror("metrics output"); + exit(4); + } + if (fprintf(output, + "{\"count\":%" PRIu64 ",\"timestamp_ns\":%" PRIu64 "}\n", + iterations, monotonic_ns()) < 0 || fclose(output) != 0) { + perror("metrics output"); + exit(5); + } + if (rename(replacement, path) != 0) { + perror("rename metrics"); + exit(6); + } +} + +int main(int argc, char **argv) +{ + struct sigaction action = {0}; + uint64_t iterations = 0U; + + if (argc != 2) { + fprintf(stderr, "usage: %s \n", argv[0]); + return 1; + } + action.sa_handler = stop_workload; + if (sigemptyset(&action.sa_mask) != 0 || + sigaction(SIGTERM, &action, NULL) != 0 || + sigaction(SIGINT, &action, NULL) != 0) { + perror("sigaction"); + return 2; + } + write_metrics(argv[1], iterations); + printf("{\"language\":\"native\",\"pid\":%ld}\n", (long)getpid()); + fflush(stdout); + while (running != 0) { + sink = xprobe_native_hot_loop(sink + iterations + 1U); + ++iterations; + if (iterations % 32U == 0U) { + write_metrics(argv[1], iterations); + } + } + write_metrics(argv[1], iterations); + return 0; +} diff --git a/benchmarks/cpu-inventory/python_workload.py b/benchmarks/cpu-inventory/python_workload.py new file mode 100644 index 0000000..ec0f96d --- /dev/null +++ b/benchmarks/cpu-inventory/python_workload.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +import gc +import json +import pathlib +import sys +import time + + +def write_metrics(path: pathlib.Path, count: int) -> None: + replacement = path.with_name(f"{path.name}.next") + replacement.write_text( + json.dumps({"count": count, "timestamp_ns": time.monotonic_ns()}) + ) + replacement.replace(path) + + +def python_leaf(seed: int) -> int: + return sum((seed + index) & 0xFFFF for index in range(4000)) + + +def python_hot_loop(seed: int) -> int: + return python_leaf(seed) ^ python_leaf(seed + 1) + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit("usage: python_workload.py ") + metrics = pathlib.Path(sys.argv[1]) + count = 0 + value = 0 + write_metrics(metrics, count) + print( + json.dumps( + { + "language": "python", + "pid": __import__("os").getpid(), + "python": sys.version.split()[0], + "perf_map": sys._xoptions.get("perf") is not None, + }, + sort_keys=True, + ), + flush=True, + ) + while True: + value ^= python_hot_loop(value) + count += 1 + if count % 64 == 0: + cycle: list[object] = [] + cycle.append(cycle) + del cycle + gc.collect() + write_metrics(metrics, count) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/cpu-inventory/run.py b/benchmarks/cpu-inventory/run.py new file mode 100644 index 0000000..aedef0b --- /dev/null +++ b/benchmarks/cpu-inventory/run.py @@ -0,0 +1,600 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import pathlib +import shutil +import subprocess +import tempfile +import time +from collections.abc import Callable + + +DEFAULT_SECONDS = 2 +FREQUENCY_HZ = 199 + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--xprobe", type=pathlib.Path, default="target/debug/xprobe") + parser.add_argument("--python", default="/usr/bin/python3") + parser.add_argument("--perf", default="perf") + parser.add_argument("--py-spy", default="py-spy") + parser.add_argument("--seconds", type=int, default=DEFAULT_SECONDS) + args = parser.parse_args() + if args.seconds < 1: + raise SystemExit("--seconds must be at least 1") + + workspace = pathlib.Path(__file__).resolve().parents[2] + xprobe = require_executable(args.xprobe) + python = require_executable(pathlib.Path(args.python)) + perf = require_command(args.perf) + py_spy = require_command(args.py_spy) + require_cpython_perf(python) + + with tempfile.TemporaryDirectory(prefix="xprobe-cpu-inventory-") as directory: + output = pathlib.Path(directory) + native = output / "native-workload" + resource_runner = output / "resource-runner" + compile_fixture(workspace / "benchmarks/cpu-inventory/native_workload.c", native) + compile_fixture( + workspace / "benchmarks/cuda-aggregate/resource_runner.c", resource_runner + ) + report = run_benchmark( + workspace, + output, + xprobe, + python, + perf, + py_spy, + native, + resource_runner, + args.seconds, + ) + print(json.dumps(report, sort_keys=True)) + + +def run_benchmark( + workspace: pathlib.Path, + output: pathlib.Path, + xprobe: pathlib.Path, + python: pathlib.Path, + perf: pathlib.Path, + py_spy: pathlib.Path, + native: pathlib.Path, + resource_runner: pathlib.Path, + seconds: int, +) -> dict: + native_cases: dict[str, dict] = {} + native_cases["baseline"] = run_case( + "native-baseline", [str(native)], output, resource_runner, ["sleep", str(seconds)] + )[0] + native_cpu, native_inventory = run_case( + "native-xprobe-cpu", + [str(native)], + output, + resource_runner, + cpu_sample_command(xprobe, seconds), + prepare=lambda pid: validate_cpu(xprobe, pid), + ) + native_cases["xprobe_cpu"] = native_cpu + native_cases["xprobe_cpu"]["quality"] = cpu_quality(native_inventory) + perf_data = output / "native-perf.data" + native_perf, _ = run_case( + "native-perf", + [str(native)], + output, + resource_runner, + [ + str(perf), + "record", + "--quiet", + "--frequency", + str(FREQUENCY_HZ), + "--call-graph", + "dwarf", + "--pid", + "{pid}", + "--output", + str(perf_data), + "--", + "sleep", + str(seconds), + ], + artifact=perf_data, + ) + native_perf["quality"] = perf_quality(perf, perf_data) + native_cases["perf"] = native_perf + syscall_case, syscall_inventory = run_case( + "native-xprobe-syscalls", + [str(native)], + output, + resource_runner, + syscall_command(xprobe, seconds), + prepare=lambda pid: validate_syscalls(xprobe, pid), + ) + syscall_case["quality"] = syscall_quality(syscall_inventory) + native_cases["xprobe_syscalls"] = syscall_case + + entry, returned = native_selector_hints(native_inventory) + exact_case, exact_result = run_case( + "native-xprobe-exact", + [str(native)], + output, + resource_runner, + exact_command(xprobe, entry, returned, "stack-nested", seconds), + prepare=lambda pid: validate_pair(xprobe, pid, entry, returned, "stack-nested"), + ) + exact_case["quality"] = exact_quality(exact_result) + native_cases["xprobe_exact"] = exact_case + + python_target = [ + str(python), + "-X", + "perf", + "-u", + str(workspace / "benchmarks/cpu-inventory/python_workload.py"), + ] + python_cases: dict[str, dict] = {} + python_cases["baseline"] = run_case( + "python-baseline", python_target, output, resource_runner, ["sleep", str(seconds)] + )[0] + python_cpu, python_inventory = run_case( + "python-xprobe-cpu", + python_target, + output, + resource_runner, + cpu_sample_command(xprobe, seconds), + prepare=lambda pid: validate_cpu(xprobe, pid), + ) + python_cpu["quality"] = cpu_quality(python_inventory) + python_cases["xprobe_cpu"] = python_cpu + py_spy_data = output / "python-py-spy.txt" + python_py_spy, _ = run_case( + "python-py-spy", + python_target, + output, + resource_runner, + [ + str(py_spy), + "record", + "--pid", + "{pid}", + "--duration", + str(seconds), + "--rate", + str(FREQUENCY_HZ), + "--format", + "raw", + "--output", + str(py_spy_data), + ], + artifact=py_spy_data, + ) + python_py_spy["quality"] = py_spy_quality(py_spy_data) + python_cases["py_spy"] = python_py_spy + gc_case, gc_result = run_case( + "python-xprobe-gc", + python_target, + output, + resource_runner, + exact_command(xprobe, "python:gc_start", "python:gc_end", "exact", seconds), + prepare=lambda pid: validate_pair( + xprobe, pid, "python:gc_start", "python:gc_end", "exact" + ), + ) + gc_case["quality"] = exact_quality(gc_result) + python_cases["xprobe_gc"] = gc_case + + assert_cpu_inventory(native_inventory, require_python=False) + assert_cpu_inventory(python_inventory, require_python=True) + assert syscall_inventory["ok"] is True + assert syscall_inventory["collection"]["dropped_aggregates"] == 0 + assert exact_result["measurement"]["samples"]["matched"] > 0 + assert gc_result["measurement"]["samples"]["matched"] > 0 + + return { + "schema_version": "2.0", + "ok": True, + "environment": { + "kernel": os.uname().release, + "python": subprocess.check_output( + [str(python), "--version"], text=True, stderr=subprocess.STDOUT + ).strip(), + "perf": tool_version(perf, ["--version"]), + "py_spy": tool_version(py_spy, ["--version"]), + "effective_uid": os.geteuid(), + }, + "workload": { + "seconds_per_case": seconds, + "frequency_hz": FREQUENCY_HZ, + "fresh_target_per_case": True, + "profilers_coexist": False, + }, + "native": summarize_cases(native_cases), + "python": summarize_cases(python_cases), + "interpretation": "workload-specific observations; no profiler is universally faster", + } + + +def run_case( + name: str, + target_prefix: list[str], + output: pathlib.Path, + resource_runner: pathlib.Path, + profiler_template: list[str], + prepare: Callable[[int], dict] | None = None, + artifact: pathlib.Path | None = None, +) -> tuple[dict, dict | None]: + metrics = output / f"{name}-target.json" + target = subprocess.Popen( + [*target_prefix, str(metrics)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + assert target.stdout is not None + ready_line = target.stdout.readline() + if not ready_line: + stderr = target.stderr.read() if target.stderr else "" + raise AssertionError(f"{name} target exited before readiness: {stderr}") + metadata = json.loads(ready_line) + wait_for_metrics(metrics) + validation = prepare(target.pid) if prepare else None + before = read_metrics(metrics) + resource_metrics = output / f"{name}-resources.txt" + command = [argument.format(pid=target.pid) for argument in profiler_template] + completed = subprocess.run( + [str(resource_runner), str(resource_metrics), str(target.pid), *command], + check=False, + capture_output=True, + text=True, + ) + after = read_metrics(metrics) + if completed.returncode != 0: + raise AssertionError( + f"{name} failed with {completed.returncode}:\n" + f"stdout={completed.stdout}\nstderr={completed.stderr}" + ) + result = json.loads(completed.stdout) if completed.stdout.strip() else None + count = after["count"] - before["count"] + elapsed_ns = after["timestamp_ns"] - before["timestamp_ns"] + if count <= 0 or elapsed_ns <= 0: + raise AssertionError({"name": name, "before": before, "after": after}) + resources = read_resource_metrics(resource_metrics) + case = { + "target": metadata, + "throughput_iterations_per_second": count * 1_000_000_000 / elapsed_ns, + "collector": { + "wall_ns": resources["wall_ns"], + "cpu_us": resources["user_us"] + resources["system_us"], + "user_us": resources["user_us"], + "system_us": resources["system_us"], + "peak_rss_kib": resources["max_rss_kib"], + }, + "target_memory": { + "start_rss_kib": resources["target_start_rss_kib"], + "peak_rss_kib": resources["target_peak_rss_kib"], + "growth_kib": resources["target_peak_rss_kib"] + - resources["target_start_rss_kib"], + }, + "artifact_bytes": artifact.stat().st_size if artifact else len(completed.stdout), + } + if validation is not None: + if validation.get("valid") is not True: + raise AssertionError({"name": name, "validation": validation}) + case["validation"] = { + "valid": True, + "target": validation["target"], + } + return case, result + finally: + target.terminate() + try: + target.wait(timeout=5) + except subprocess.TimeoutExpired: + target.kill() + target.wait(timeout=5) + + +def cpu_sample_command(xprobe: pathlib.Path, seconds: int) -> list[str]: + return [ + str(xprobe), + "measure", + "--pid", + "{pid}", + "--cpu-sample", + "--duration-ms", + str(seconds * 1000), + "--frequency-hz", + str(FREQUENCY_HZ), + "--max-samples", + str(seconds * FREQUENCY_HZ * 4), + "--max-groups", + "512", + "--json", + "--non-interactive", + "--no-color", + ] + + +def syscall_command(xprobe: pathlib.Path, seconds: int) -> list[str]: + return [ + str(xprobe), + "measure", + "--pid", + "{pid}", + "--syscall-aggregate", + "--duration-ms", + str(seconds * 1000), + "--max-groups", + "256", + "--json", + "--non-interactive", + "--no-color", + ] + + +def exact_command( + xprobe: pathlib.Path, start: str, end: str, policy: str, seconds: int +) -> list[str]: + return [ + str(xprobe), + "measure", + "--pid", + "{pid}", + "--from", + start, + "--to", + end, + "--match", + policy, + "--duration-ms", + str(seconds * 1000), + "--max-events", + "50000", + "--json", + "--non-interactive", + "--no-color", + ] + + +def validate_cpu(xprobe: pathlib.Path, pid: int) -> dict: + return run_json( + [str(xprobe), "validate", "--pid", str(pid), "--cpu-sample", *json_flags()] + ) + + +def validate_syscalls(xprobe: pathlib.Path, pid: int) -> dict: + return run_json( + [ + str(xprobe), + "validate", + "--pid", + str(pid), + "--syscall-aggregate", + *json_flags(), + ] + ) + + +def validate_pair( + xprobe: pathlib.Path, pid: int, start: str, end: str, policy: str +) -> dict: + return run_json( + [ + str(xprobe), + "validate", + "--pid", + str(pid), + "--from", + start, + "--to", + end, + "--match", + policy, + *json_flags(), + ] + ) + + +def json_flags() -> list[str]: + return ["--json", "--non-interactive", "--no-color"] + + +def run_json(command: list[str]) -> dict: + completed = subprocess.run(command, check=False, capture_output=True, text=True) + if completed.returncode != 0: + raise AssertionError( + f"command failed with {completed.returncode}: {' '.join(command)}\n" + f"stdout={completed.stdout}\nstderr={completed.stderr}" + ) + return json.loads(completed.stdout) + + +def native_selector_hints(inventory: dict) -> tuple[str, str]: + for hotspot in inventory["inventory"]["hotspots"]: + if hotspot["frame"]["symbol"] == "xprobe_native_hot_loop": + entry = hotspot.get("entry_selector_hint") + returned = hotspot.get("return_selector_hint") + if entry and returned: + return entry, returned + raise AssertionError("CPU inventory did not produce selectors for xprobe_native_hot_loop") + + +def cpu_quality(result: dict) -> dict: + collection = result["collection"] + symbols = result["symbolization"] + observed = collection["observed_samples"] + total_frames = symbols["total_frames"] + resolved = symbols["resolved_native_frames"] + symbols["resolved_python_frames"] + return { + "completeness": collection["completeness"], + "observed_samples": observed, + "grouped_samples": collection["grouped_samples"], + "lost_samples": collection["lost_samples"], + "sample_capacity": collection["sample_capacity"], + "group_capacity": collection["group_capacity"], + "groups": collection["groups"], + "stack_coverage": collection["grouped_samples"] / observed if observed else 0.0, + "symbol_coverage": resolved / total_frames if total_frames else 0.0, + "python_status": symbols["python_status"], + "truncated_stacks": collection["truncated_stacks"], + } + + +def syscall_quality(result: dict) -> dict: + collection = result["collection"] + return { + "completeness": collection["completeness"], + "observed_entries": collection["observed_entries"], + "matched_exits": collection["matched_exits"], + "unmatched_exits": collection["unmatched_exits"], + "inflight_at_end": collection["inflight_at_end"], + "dropped_aggregates": collection["dropped_aggregates"], + "groups": collection["groups"], + "group_capacity": collection["group_capacity"], + } + + +def exact_quality(result: dict) -> dict: + samples = result["measurement"]["samples"] + return { + "completeness": result["collection"]["completeness"], + "matched_samples": samples["matched"], + "unmatched_start_samples": samples["unmatched_start"], + "unmatched_end_samples": samples["unmatched_end"], + "ambiguous_samples": samples["ambiguous"], + "dropped_events": result["collection"]["dropped_events"], + "correlation_confidence": result["correlation"]["confidence"], + } + + +def perf_quality(perf: pathlib.Path, data: pathlib.Path) -> dict: + completed = subprocess.run( + [str(perf), "script", "--input", str(data)], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode != 0: + raise AssertionError(f"perf script failed:\n{completed.stderr}") + samples = sum(1 for line in completed.stdout.splitlines() if line and not line[0].isspace()) + return {"decoded_samples": samples, "lost_samples": None} + + +def py_spy_quality(path: pathlib.Path) -> dict: + samples = 0 + stacks = 0 + for line in path.read_text().splitlines(): + stack, separator, count = line.rpartition(" ") + if not separator or not stack: + raise AssertionError(f"malformed py-spy raw record: {line}") + samples += int(count) + stacks += 1 + return {"decoded_samples": samples, "stack_groups": stacks, "lost_samples": None} + + +def assert_cpu_inventory(result: dict, require_python: bool) -> None: + assert result["ok"] is True, result + assert result["status"] == "completed", result + assert result["collection"]["completeness"] == "complete", result + assert result["collection"]["observed_samples"] > 0, result + assert result["collection"]["lost_samples"] == 0, result + assert result["collection"]["grouped_samples"] > 0, result + assert result["symbolization"]["resolved_native_frames"] > 0, result + if require_python: + assert result["symbolization"]["python_status"] == "active", result + assert result["symbolization"]["resolved_python_frames"] > 0, result + + +def summarize_cases(cases: dict[str, dict]) -> dict: + baseline = cases["baseline"]["throughput_iterations_per_second"] + for case in cases.values(): + observed = case["throughput_iterations_per_second"] + case["overhead_ratio"] = baseline / observed + return cases + + +def read_metrics(path: pathlib.Path) -> dict[str, int]: + return {key: int(value) for key, value in json.loads(path.read_text()).items()} + + +def read_resource_metrics(path: pathlib.Path) -> dict[str, int]: + return { + key: int(value) + for key, value in (line.split() for line in path.read_text().splitlines()) + } + + +def wait_for_metrics(path: pathlib.Path) -> None: + deadline = time.monotonic() + 10 + while not path.is_file(): + if time.monotonic() >= deadline: + raise TimeoutError(f"timed out waiting for {path}") + time.sleep(0.01) + + +def compile_fixture(source: pathlib.Path, output: pathlib.Path) -> None: + subprocess.run( + [ + "cc", + "-std=c11", + "-O2", + "-g", + "-fno-omit-frame-pointer", + "-no-pie", + "-Wall", + "-Wextra", + "-Wpedantic", + "-Werror", + str(source), + "-o", + str(output), + ], + check=True, + ) + + +def require_executable(path: pathlib.Path) -> pathlib.Path: + resolved = path.resolve() + if not resolved.is_file() or not os.access(resolved, os.X_OK): + raise SystemExit(f"required executable is unavailable: {path}") + return resolved + + +def require_command(command: str) -> pathlib.Path: + resolved = shutil.which(command) + if resolved is None: + raise SystemExit(f"required benchmark tool is unavailable: {command}") + return pathlib.Path(resolved).resolve() + + +def require_cpython_perf(python: pathlib.Path) -> None: + completed = subprocess.run( + [ + str(python), + "-X", + "perf", + "-c", + "import sys; assert sys._xoptions.get('perf') is True", + ], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode != 0: + raise SystemExit( + f"Python benchmark requires CPython -X perf support: {completed.stderr}" + ) + + +def tool_version(tool: pathlib.Path, arguments: list[str]) -> str: + return subprocess.check_output( + [str(tool), *arguments], text=True, stderr=subprocess.STDOUT + ).strip() + + +if __name__ == "__main__": + main() diff --git a/justfile b/justfile index 99350a3..9fb10c0 100644 --- a/justfile +++ b/justfile @@ -102,6 +102,9 @@ benchmark-multiprocess: benchmark-pytorch: build if [[ -n "${PYTORCH_ENV:-}" ]]; then python3 benchmarks/pytorch/run.py --image "{{cuda12_devel_image}}" --pytorch-env "${PYTORCH_ENV}"; else python3 benchmarks/pytorch/run.py --image "{{pytorch_image}}"; fi +benchmark-cpu: build + python3 benchmarks/cpu-inventory/run.py + fmt: cargo fmt --all From 616170da6391b23cb2dc975aaf300107f1a9f899 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 1 Aug 2026 05:22:46 +0800 Subject: [PATCH 08/15] =?UTF-8?q?=F0=9F=91=B7=20ci:=20gate=20releases=20on?= =?UTF-8?q?=20hardware=20profiling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 12 ++++++------ .github/workflows/hardware.yml | 28 +++++++++++++++++++++++++- .github/workflows/release.yml | 36 +++++++++++++++++++++++++++------- docs/benchmarks.md | 23 ++++++++++++++++++++++ docs/development.md | 22 +++++++++++++++++++++ 5 files changed, 107 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1501d4f..37f7626 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@stable with: components: clippy,rustfmt @@ -38,7 +38,7 @@ jobs: container: image: ${{ matrix.image }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install native build tools run: >- apt-get update && apt-get install -y --no-install-recommends @@ -62,7 +62,7 @@ jobs: if readelf -d build/cuda/cupti/libxprobe-cupti.so | grep -E '\((RPATH|RUNPATH)\)'; then exit 1 fi - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: xprobe-cupti-cuda${{ matrix.cuda_major }}-linux-x86_64 path: build/cuda/cupti/libxprobe-cupti.so @@ -75,7 +75,7 @@ jobs: runs-on: ubuntu-22.04 timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install build tools run: | sudo apt-get update @@ -86,11 +86,11 @@ jobs: run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: name: xprobe-cupti-cuda12-linux-x86_64 path: build/cuda12/cupti - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: name: xprobe-cupti-cuda13-linux-x86_64 path: build/cuda13/cupti diff --git a/.github/workflows/hardware.yml b/.github/workflows/hardware.yml index 489a920..6848f9e 100644 --- a/.github/workflows/hardware.yml +++ b/.github/workflows/hardware.yml @@ -9,13 +9,25 @@ permissions: jobs: gpu-and-bpf: runs-on: [self-hosted, linux, x64, nvidia] + timeout-minutes: 120 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@stable with: components: clippy,rustfmt - uses: taiki-e/install-action@just + - name: Verify CPU profiler prerequisites + run: | + command -v perf + command -v py-spy + sudo -n true + /usr/bin/python3 -X perf -c \ + "import sys; assert sys._xoptions.get('perf') is True" - run: just test-bpf-live + - name: Test CPU and Python sampling + run: >- + sudo --preserve-env=PATH,CARGO_HOME,RUSTUP_HOME + "$(command -v just)" test-cpu-live - run: just test-cupti-live-cuda12 - run: just test-nvtx-live-cuda12 - run: just test-cupti-live-cuda12-min @@ -31,3 +43,17 @@ jobs: - run: just benchmark-aggregate - run: just benchmark-multiprocess - run: just benchmark-pytorch + - name: Benchmark CPU inventory + run: >- + sudo --preserve-env=PATH + /usr/bin/python3 benchmarks/cpu-inventory/run.py + --xprobe "$PWD/target/debug/xprobe" + --python /usr/bin/python3 + --perf "$(command -v perf)" + --py-spy "$(command -v py-spy)" + | tee cpu-inventory-benchmark.json + - uses: actions/upload-artifact@v7 + with: + name: cpu-inventory-benchmark + path: cpu-inventory-benchmark.json + if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 869c69d..d97efcb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,10 +6,32 @@ on: - "v*" permissions: + actions: read contents: write jobs: + hardware-gate: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + - name: Require successful hardware validation for this commit + env: + GH_TOKEN: ${{ github.token }} + run: | + commit=$(git rev-parse HEAD) + runs=$(gh api --method GET \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/hardware.yml/runs" \ + -f head_sha="$commit" \ + -f status=success \ + -f per_page=100 \ + --jq '.total_count') + if [ "$runs" -lt 1 ]; then + echo "No successful Hardware Integration run exists for $commit" >&2 + exit 1 + fi + cupti-agent: + needs: hardware-gate runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -22,7 +44,7 @@ jobs: container: image: ${{ matrix.image }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install build tools env: DEBIAN_FRONTEND: noninteractive @@ -45,7 +67,7 @@ jobs: grep -E '\((RPATH|RUNPATH)\)'; then exit 1 fi - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: xprobe-cupti-cuda${{ matrix.cuda_major }}-linux-x86_64 path: build/cuda/cupti/libxprobe-cupti.so @@ -55,7 +77,7 @@ jobs: needs: cupti-agent runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install build tools run: | sudo apt-get update @@ -66,11 +88,11 @@ jobs: run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: name: xprobe-cupti-cuda12-linux-x86_64 path: build/cuda12/cupti - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: name: xprobe-cupti-cuda13-linux-x86_64 path: build/cuda13/cupti @@ -78,11 +100,11 @@ jobs: run: scripts/package-release.sh - name: Verify release installation run: tests/install/test_install.sh dist/xprobe-*-linux-x86_64.tar.gz - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: xprobe-linux-x86_64 path: dist/*.tar.gz* - - uses: softprops/action-gh-release@v2 + - uses: softprops/action-gh-release@v3 with: files: dist/*.tar.gz* generate_release_notes: true diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 162789c..2f5a5fd 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1,5 +1,28 @@ # Precision and overhead benchmarks +Run the CPU and Python inventory benchmark on a Linux host with `perf`, +`py-spy`, CPython 3.12 or newer with `-X perf`, perf-event access, and eBPF +attach permission: + +```bash +just benchmark-cpu +``` + +The benchmark uses a fresh target for every case. Native and Python workloads +are each measured without a profiler, with bounded xprobe CPU sampling, and +with their corresponding broad profiler (`perf` or `py-spy`). A native syscall +inventory is measured separately. The native exact measurement uses selector +hints emitted by the sampled inventory; the Python exact measurement uses GC +USDT only after validation proves that capability. Profilers never coexist. + +The JSON report includes throughput, collector CPU and peak RSS, target RSS, +artifact size, sample loss, stack and symbol coverage, retained capacity, and +exact-correlation quality. It rejects incomplete or lossy xprobe captures, +missing native or Python symbols, malformed comparison artifacts, and empty +exact measurements. It does not enforce an overhead winner: the observations +are workload-specific and do not establish that any profiler is universally +faster. + Run the reproducible GPU benchmark with the pinned CUDA devel image: ```bash diff --git a/docs/development.md b/docs/development.md index 489b710..12b029f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -20,6 +20,11 @@ Agents without a GPU in pinned NVIDIA devel images, checks their SONAMEs, and rejects ABI-only output or build-time RPATHs. Live CUDA behavior remains a hardware test on an NVIDIA runner. +The self-hosted hardware runner must use Actions Runner 2.329.0 or newer and +provide passwordless `sudo`, `perf`, `py-spy`, and `/usr/bin/python3` with +`-X perf`. These are host profiler prerequisites, not release archive +dependencies. + ## Release packaging A release archive requires one Agent built against each supported CUPTI major: @@ -48,6 +53,11 @@ the public archive and checksum again, repeats the installation test, and inspects every shipped ELF. This final gate verifies the artifact users can actually download rather than the workflow's local copy. +Before creating a tag, manually run `Hardware Integration` for the exact commit +to be tagged. The release workflow queries GitHub Actions for a successful run +whose `head_sha` matches that commit and fails before building artifacts when +the hardware result is absent. + ## eBPF tests Compile the BPF object without attaching it: @@ -82,6 +92,18 @@ The target interpreter must provide perf trampoline support for the Python case; unsupported interpreters remain a valid native-only product path but are not a substitute for this live gate. +Run the comparative CPU/Python perturbation benchmark after changing CPU +sampling, symbolization, syscall aggregation, Python GC probes, or the Skill's +broad-to-narrow route: + +```bash +just benchmark-cpu +``` + +The benchmark requires `perf` and `py-spy` and may require root on hosts whose +perf-event or eBPF policy denies attachment. See `docs/benchmarks.md` for its +reported metrics and interpretation. + Resolve real CPython, native extension, and libtorch C++ symbols with a local Python environment containing PyTorch: From 4927e7d3ae8c552e39c1959f77201a590f232405 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 1 Aug 2026 05:23:52 +0800 Subject: [PATCH 09/15] =?UTF-8?q?=F0=9F=94=96=20chore:=20prepare=20version?= =?UTF-8?q?=200.5.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeLists.txt | 2 +- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- README.md | 2 +- docs/agent-integration.md | 2 +- docs/installation.md | 12 ++++++------ install.sh | 4 ++-- skills/xprobe-measure-latency/references/setup.md | 8 ++++---- tests/agent-contract/test_contract.py | 2 +- 9 files changed, 23 insertions(+), 23 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 34eb397..813d883 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.20) -project(xprobe VERSION 0.4.1 LANGUAGES C) +project(xprobe VERSION 0.5.0 LANGUAGES C) include(CTest) diff --git a/Cargo.lock b/Cargo.lock index 487ae71..e7e2def 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -732,7 +732,7 @@ dependencies = [ [[package]] name = "xprobe-cli" -version = "0.4.1" +version = "0.5.0" dependencies = [ "clap", "serde_json", @@ -745,7 +745,7 @@ dependencies = [ [[package]] name = "xprobe-collector" -version = "0.4.1" +version = "0.5.0" dependencies = [ "libbpf-rs", "perf-event-open", @@ -755,7 +755,7 @@ dependencies = [ [[package]] name = "xprobe-core" -version = "0.4.1" +version = "0.5.0" dependencies = [ "cpp_demangle", "nix", @@ -766,7 +766,7 @@ dependencies = [ [[package]] name = "xprobe-correlator" -version = "0.4.1" +version = "0.5.0" dependencies = [ "regex", "serde_json", @@ -775,7 +775,7 @@ dependencies = [ [[package]] name = "xprobe-exporter" -version = "0.4.1" +version = "0.5.0" dependencies = [ "serde_json", "xprobe-protocol", @@ -783,7 +783,7 @@ dependencies = [ [[package]] name = "xprobe-protocol" -version = "0.4.1" +version = "0.5.0" dependencies = [ "schemars", "serde", diff --git a/Cargo.toml b/Cargo.toml index 29ee202..e28ad6b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.4.1" +version = "0.5.0" edition = "2024" license = "Apache-2.0" repository = "https://github.com/itdevwu/xprobe" diff --git a/README.md b/README.md index 9be70b4..e950b14 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ only installation action required from the user: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.4.1/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.5.0/skills/xprobe-measure-latency \ --global ``` diff --git a/docs/agent-integration.md b/docs/agent-integration.md index 4f1ad45..a9fa126 100644 --- a/docs/agent-integration.md +++ b/docs/agent-integration.md @@ -17,7 +17,7 @@ repairs the matching xprobe CLI itself: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.4.1/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.5.0/skills/xprobe-measure-latency \ --global ``` diff --git a/docs/installation.md b/docs/installation.md index 3faeff3..32bf86d 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -13,7 +13,7 @@ user needs to run: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.4.1/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.5.0/skills/xprobe-measure-latency \ --global ``` @@ -28,7 +28,7 @@ The versioned bootstrap installs to `~/.local` without root access: ```bash curl --proto '=https' --tlsv1.2 -fsSL \ - https://raw.githubusercontent.com/itdevwu/xprobe/v0.4.1/install.sh | sh + https://raw.githubusercontent.com/itdevwu/xprobe/v0.5.0/install.sh | sh ``` The bootstrap downloads the release archive and its SHA256 file, verifies the @@ -47,7 +47,7 @@ prefix, download the script and pass `--prefix`: ```bash curl --proto '=https' --tlsv1.2 -fsSLO \ - https://raw.githubusercontent.com/itdevwu/xprobe/v0.4.1/install.sh + https://raw.githubusercontent.com/itdevwu/xprobe/v0.5.0/install.sh sh install.sh --prefix /opt/xprobe ``` @@ -59,7 +59,7 @@ script with `sudo`. The installer never elevates privileges itself. For a fully explicit archive workflow: ```bash -version=0.4.1 +version=0.5.0 base=https://github.com/itdevwu/xprobe/releases/download/v$version archive=xprobe-$version-linux-x86_64.tar.gz @@ -92,7 +92,7 @@ missing or damaged, manually install the complete version-matched directory: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.4.1/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.5.0/skills/xprobe-measure-latency \ --global ``` @@ -101,7 +101,7 @@ installation names the target explicitly: ```bash npx --yes skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.4.1/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.5.0/skills/xprobe-measure-latency \ --agent codex --global --copy --yes ``` diff --git a/install.sh b/install.sh index e631737..cddd7fa 100755 --- a/install.sh +++ b/install.sh @@ -2,7 +2,7 @@ set -eu repository=${XPROBE_REPOSITORY:-itdevwu/xprobe} -version=${XPROBE_VERSION:-0.4.1} +version=${XPROBE_VERSION:-0.5.0} if [ -n "${XPROBE_PREFIX:-}" ]; then prefix=$XPROBE_PREFIX elif [ -n "${HOME:-}" ]; then @@ -19,7 +19,7 @@ Install a released xprobe binary and its CUDA Agents. Usage: install.sh [--version VERSION] [--prefix DIR] [--uninstall] Options: - --version VERSION Release to install (default: 0.4.1) + --version VERSION Release to install (default: 0.5.0) --prefix DIR Installation prefix (default: $HOME/.local) --uninstall Remove xprobe from the selected prefix -h, --help Show this help diff --git a/skills/xprobe-measure-latency/references/setup.md b/skills/xprobe-measure-latency/references/setup.md index cc20863..dc6c130 100644 --- a/skills/xprobe-measure-latency/references/setup.md +++ b/skills/xprobe-measure-latency/references/setup.md @@ -23,7 +23,7 @@ installing under `~/.local`: ```bash curl --proto '=https' --tlsv1.2 -fsSL \ - https://raw.githubusercontent.com/itdevwu/xprobe/v0.4.1/install.sh \ + https://raw.githubusercontent.com/itdevwu/xprobe/v0.5.0/install.sh \ -o /tmp/xprobe-install.sh sh /tmp/xprobe-install.sh export PATH="$HOME/.local/bin:$PATH" @@ -46,7 +46,7 @@ host glibc instead. This is a local-use fallback, not permission to weaken the release package's `GLIBC_2.34` ceiling. ```bash -git clone --depth 1 --branch v0.4.1 https://github.com/itdevwu/xprobe.git +git clone --depth 1 --branch v0.5.0 https://github.com/itdevwu/xprobe.git cd xprobe mamba env create --file environment.yml mamba run -n xprobe-dev just build @@ -89,7 +89,7 @@ release directory through the Agent Skills CLI: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.4.1/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.5.0/skills/xprobe-measure-latency \ --global ``` @@ -97,7 +97,7 @@ For non-interactive automation, select the host explicitly: ```bash npx --yes skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.4.1/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.5.0/skills/xprobe-measure-latency \ --agent codex --global --copy --yes ``` diff --git a/tests/agent-contract/test_contract.py b/tests/agent-contract/test_contract.py index 9e5d350..13c16af 100755 --- a/tests/agent-contract/test_contract.py +++ b/tests/agent-contract/test_contract.py @@ -222,7 +222,7 @@ def check_skill(workspace: pathlib.Path) -> None: ): assert required in normalized_cli_contract for required in ( - "v0.4.1/install.sh", + "v0.5.0/install.sh", "xprobe `0.4.x`", "npx skills@1 add", "xprobe --version", From 86fa2e897e50a2353c6fce0c38f18b0313ac495b Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 1 Aug 2026 05:25:21 +0800 Subject: [PATCH 10/15] =?UTF-8?q?=F0=9F=90=9B=20fix:=20use=20portable=20pe?= =?UTF-8?q?rf=20frequency=20option?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- benchmarks/cpu-inventory/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/cpu-inventory/run.py b/benchmarks/cpu-inventory/run.py index aedef0b..768cefc 100644 --- a/benchmarks/cpu-inventory/run.py +++ b/benchmarks/cpu-inventory/run.py @@ -89,7 +89,7 @@ def run_benchmark( str(perf), "record", "--quiet", - "--frequency", + "--freq", str(FREQUENCY_HZ), "--call-graph", "dwarf", From 068bebc08151b28e762c6919cf54346645a5ecac Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 1 Aug 2026 05:28:07 +0800 Subject: [PATCH 11/15] =?UTF-8?q?=F0=9F=90=9B=20fix:=20narrow=20CPU=20hint?= =?UTF-8?q?s=20to=20symbol=20boundaries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- xprobe/core/src/cpu_sampling_analysis.rs | 49 ++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/xprobe/core/src/cpu_sampling_analysis.rs b/xprobe/core/src/cpu_sampling_analysis.rs index f79a915..13758c7 100644 --- a/xprobe/core/src/cpu_sampling_analysis.rs +++ b/xprobe/core/src/cpu_sampling_analysis.rs @@ -420,12 +420,17 @@ fn selector_hints(frame: &CpuStackFrame) -> (Option, Option) { if frame.language != CpuFrameLanguage::Native { return (None, None); } - let (Some(path), Some(offset)) = (&frame.module_path, frame.file_offset) else { + let (Some(path), Some(offset), Some(symbol_offset)) = + (&frame.module_path, frame.file_offset, frame.symbol_offset) + else { + return (None, None); + }; + let Some(symbol_start) = offset.checked_sub(symbol_offset) else { return (None, None); }; ( - Some(format!("uprobe:{path}:+0x{offset:x}:entry")), - Some(format!("uprobe:{path}:+0x{offset:x}:return")), + Some(format!("uprobe:{path}:+0x{symbol_start:x}:entry")), + Some(format!("uprobe:{path}:+0x{symbol_start:x}:return")), ) } @@ -798,6 +803,44 @@ mod tests { assert!(find_perf_symbol(&symbols, 0x7f00_0030).is_none()); } + #[test] + fn selector_hints_use_the_resolved_symbol_start() { + let frame = CpuStackFrame { + address: 0x7f00_0042, + module_path: Some("/tmp/native workload".to_owned()), + build_id: None, + file_offset: Some(0x1642), + symbol: Some("hot_loop".to_owned()), + symbol_offset: Some(0x42), + language: CpuFrameLanguage::Native, + source_path: None, + line: None, + }; + assert_eq!( + selector_hints(&frame), + ( + Some("uprobe:/tmp/native workload:+0x1600:entry".to_owned()), + Some("uprobe:/tmp/native workload:+0x1600:return".to_owned()), + ) + ); + } + + #[test] + fn unresolved_native_frames_do_not_claim_function_boundaries() { + let frame = CpuStackFrame { + address: 0x7f00_0042, + module_path: Some("/tmp/native".to_owned()), + build_id: None, + file_offset: Some(0x1642), + symbol: None, + symbol_offset: None, + language: CpuFrameLanguage::Native, + source_path: None, + line: None, + }; + assert_eq!(selector_hints(&frame), (None, None)); + } + fn raw(addresses: &[u64]) -> RawCpuSample { RawCpuSample { thread_id: 1, From f563cb27682c268e70d01632e85085a49f3ace7e Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 1 Aug 2026 05:29:04 +0800 Subject: [PATCH 12/15] =?UTF-8?q?=F0=9F=90=9B=20fix:=20parse=20only=20xpro?= =?UTF-8?q?be=20benchmark=20JSON?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- benchmarks/cpu-inventory/run.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/benchmarks/cpu-inventory/run.py b/benchmarks/cpu-inventory/run.py index 768cefc..7a8bbd5 100644 --- a/benchmarks/cpu-inventory/run.py +++ b/benchmarks/cpu-inventory/run.py @@ -76,6 +76,7 @@ def run_benchmark( resource_runner, cpu_sample_command(xprobe, seconds), prepare=lambda pid: validate_cpu(xprobe, pid), + json_result=True, ) native_cases["xprobe_cpu"] = native_cpu native_cases["xprobe_cpu"]["quality"] = cpu_quality(native_inventory) @@ -112,6 +113,7 @@ def run_benchmark( resource_runner, syscall_command(xprobe, seconds), prepare=lambda pid: validate_syscalls(xprobe, pid), + json_result=True, ) syscall_case["quality"] = syscall_quality(syscall_inventory) native_cases["xprobe_syscalls"] = syscall_case @@ -124,6 +126,7 @@ def run_benchmark( resource_runner, exact_command(xprobe, entry, returned, "stack-nested", seconds), prepare=lambda pid: validate_pair(xprobe, pid, entry, returned, "stack-nested"), + json_result=True, ) exact_case["quality"] = exact_quality(exact_result) native_cases["xprobe_exact"] = exact_case @@ -146,6 +149,7 @@ def run_benchmark( resource_runner, cpu_sample_command(xprobe, seconds), prepare=lambda pid: validate_cpu(xprobe, pid), + json_result=True, ) python_cpu["quality"] = cpu_quality(python_inventory) python_cases["xprobe_cpu"] = python_cpu @@ -182,6 +186,7 @@ def run_benchmark( prepare=lambda pid: validate_pair( xprobe, pid, "python:gc_start", "python:gc_end", "exact" ), + json_result=True, ) gc_case["quality"] = exact_quality(gc_result) python_cases["xprobe_gc"] = gc_case @@ -225,6 +230,7 @@ def run_case( profiler_template: list[str], prepare: Callable[[int], dict] | None = None, artifact: pathlib.Path | None = None, + json_result: bool = False, ) -> tuple[dict, dict | None]: metrics = output / f"{name}-target.json" target = subprocess.Popen( @@ -257,7 +263,7 @@ def run_case( f"{name} failed with {completed.returncode}:\n" f"stdout={completed.stdout}\nstderr={completed.stderr}" ) - result = json.loads(completed.stdout) if completed.stdout.strip() else None + result = json.loads(completed.stdout) if json_result else None count = after["count"] - before["count"] elapsed_ns = after["timestamp_ns"] - before["timestamp_ns"] if count <= 0 or elapsed_ns <= 0: From 89894c3b38948b93bbad55f277af9e410c083e9d Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 1 Aug 2026 05:31:21 +0800 Subject: [PATCH 13/15] =?UTF-8?q?=F0=9F=90=9B=20fix:=20stop=20benchmark=20?= =?UTF-8?q?after=20invalid=20validation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- benchmarks/cpu-inventory/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/cpu-inventory/run.py b/benchmarks/cpu-inventory/run.py index 7a8bbd5..2ba00be 100644 --- a/benchmarks/cpu-inventory/run.py +++ b/benchmarks/cpu-inventory/run.py @@ -248,6 +248,8 @@ def run_case( metadata = json.loads(ready_line) wait_for_metrics(metrics) validation = prepare(target.pid) if prepare else None + if validation is not None and validation.get("valid") is not True: + raise AssertionError({"name": name, "validation": validation}) before = read_metrics(metrics) resource_metrics = output / f"{name}-resources.txt" command = [argument.format(pid=target.pid) for argument in profiler_template] @@ -288,8 +290,6 @@ def run_case( "artifact_bytes": artifact.stat().st_size if artifact else len(completed.stdout), } if validation is not None: - if validation.get("valid") is not True: - raise AssertionError({"name": name, "validation": validation}) case["validation"] = { "valid": True, "target": validation["target"], From 2fdff66bebe00a083ac75bce6553781cafe9d718 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 1 Aug 2026 05:36:49 +0800 Subject: [PATCH 14/15] =?UTF-8?q?=F0=9F=94=A7=20chore:=20provision=20CPU?= =?UTF-8?q?=20profiler=20tools?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/development.md | 14 +++++++------- environment.yml | 2 ++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/development.md b/docs/development.md index 12b029f..69be06b 100644 --- a/docs/development.md +++ b/docs/development.md @@ -12,13 +12,13 @@ just build just test ``` -The environment contains Clang, CMake, Ninja, pkg-config, Just, Python, and the -autotools required to compile vendored libbpf, libelf, and zlib. A system C -compiler and Linux UAPI/multiarch headers are also required. CUDA is not -installed into the Mamba environment. CI compiles CUDA 12 and CUDA 13 CUPTI -Agents without a GPU in pinned NVIDIA devel images, checks their SONAMEs, and -rejects ABI-only output or build-time RPATHs. Live CUDA behavior remains a -hardware test on an NVIDIA runner. +The environment contains Clang, CMake, Ninja, pkg-config, Just, Python, `perf`, +`py-spy`, and the autotools required to compile vendored libbpf, libelf, and +zlib. A system C compiler and Linux UAPI/multiarch headers are also required. +CUDA is not installed into the Mamba environment. CI compiles CUDA 12 and CUDA +13 CUPTI Agents without a GPU in pinned NVIDIA devel images, checks their +SONAMEs, and rejects ABI-only output or build-time RPATHs. Live CUDA behavior +remains a hardware test on an NVIDIA runner. The self-hosted hardware runner must use Actions Runner 2.329.0 or newer and provide passwordless `sudo`, `perf`, `py-spy`, and `/usr/bin/python3` with diff --git a/environment.yml b/environment.yml index d541baf..792d483 100644 --- a/environment.yml +++ b/environment.yml @@ -14,8 +14,10 @@ dependencies: - gawk>=5.3 - gettext>=0.25 - just>=1.40 + - linux-perf>=7.1 - make>=4.4 - ninja>=1.12 - pkg-config>=0.29 - python>=3.12,<3.14 + - py-spy>=0.4.2 - pyyaml>=6.0 From 03a3143af4d793eeb9dba4634de1c07c9a165dcb Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 1 Aug 2026 17:27:59 +0800 Subject: [PATCH 15/15] =?UTF-8?q?=F0=9F=90=9B=20fix:=20respect=20CPU=20sam?= =?UTF-8?q?pling=20permissions=20in=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- xprobe/cli/tests/validate.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/xprobe/cli/tests/validate.rs b/xprobe/cli/tests/validate.rs index 1eadc48..b2ec2ab 100644 --- a/xprobe/cli/tests/validate.rs +++ b/xprobe/cli/tests/validate.rs @@ -1,8 +1,8 @@ use std::process::Command; use xprobe_protocol::{ - AgentActivation, CpuSamplingValidationResult, ErrorCode, ErrorResponse, MatchPolicy, - PythonSymbolizationStatus, SyscallAggregateValidationResult, ValidationResult, + AgentActivation, CheckStatus, CpuSamplingValidationResult, ErrorCode, ErrorResponse, + MatchPolicy, PythonSymbolizationStatus, SyscallAggregateValidationResult, ValidationResult, }; #[test] @@ -50,7 +50,6 @@ fn validate_reports_environment_requirements_without_attaching() { AgentActivation::InjectionRequired ); assert!(result.requirements.target_mutation); - assert!(result.valid); assert!(result.issues.is_empty()); assert!( result @@ -107,7 +106,14 @@ fn validate_accepts_pid_scoped_cpu_sampling_without_ebpf() { assert!(output.stderr.is_empty()); let result: CpuSamplingValidationResult = serde_json::from_slice(&output.stdout).expect("stdout must contain validation JSON"); - assert!(result.valid); + assert_eq!(result.valid, result.issues.is_empty()); + if result.perf_event.status == CheckStatus::Available { + assert!(result.valid); + } else { + assert!(!result.valid); + assert_eq!(result.issues.len(), 1); + assert_eq!(result.issues[0].code, ErrorCode::PermissionDenied); + } assert!(result.requirements.needs_perf_event); assert!(!result.requirements.target_mutation); assert!(result.target_threads >= 1);