From c89d8423b1bda2f9ac610eaa70ae63df4b66034b Mon Sep 17 00:00:00 2001 From: cculin Date: Fri, 20 Nov 2026 15:15:02 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20macOS=20=E5=86=85?= =?UTF-8?q?=E5=AD=98=E5=8E=8B=E5=8A=9B=E8=AF=AF=E5=88=A4=EF=BC=9A=E6=94=B9?= =?UTF-8?q?=E8=AF=BB=E5=86=85=E6=A0=B8=E5=8E=8B=E5=8A=9B=E7=AD=89=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 现象:Mac 上运行一段时间后,「获取模型」与凭证刷新等后台任务全部失败(排队 10 秒后返回 plugin_busy),插件模型清单刷不出来,客户端按已下线的模型名请求直接报错。 原因:RuntimeCapacity 用「可用内存 < 总内存/10」判内存紧张,而 macOS 上 available 的算法是 free + inactive + purgeable − compressor。压缩内存动辄十几 GB,相减后恒为 0,于是 memory_pressure 恒为 true、background 并发被压到 0;这些机器上内核报告的压力等级其实一直是 1(正常)。 做法:monitor 新增 os_pressure_level(macOS 读 kern.memorystatus_vm_pressure_level),effective() 有等级时以等级为准(>=2 才算紧张),取不到时仍走原来的比例判据;进程/容器占用超总内存 4/5 的硬上限不变。附三条同文件单测覆盖「内核正常但可用页为 0」「内核紧张」「无内核等级」。 --- .../src/server/core/runtime_capacity/mod.rs | 55 ++++++++++++++++++- .../server/core/runtime_capacity/monitor.rs | 30 +++++++++- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/mod.rs b/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/mod.rs index ffd0c00f2..061524ce1 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/mod.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/mod.rs @@ -78,8 +78,11 @@ fn effective(settings: &CapacitySettings, resources: monitor::Resources) -> Limi let memory_bytes = if settings.memory_mib == 0 { auto_memory.max(16 * MIB) } else { settings.memory_mib.saturating_mul(MIB).min((resources.total_bytes * 3 / 4) as usize).max(16 * MIB) }; let process_ceiling = resources.total_bytes.saturating_mul(4) / 5; - let memory_pressure = resources.available_bytes < resources.total_bytes / 10 - || resources.container_bytes.unwrap_or(resources.process_bytes) >= process_ceiling; + // 有内核压力等级就以它为准:按空闲页推算在 macOS 上会因为压缩内存而长期误判。 + let memory_pressure = match resources.os_pressure_level { + Some(level) => level >= 2, + None => resources.available_bytes < resources.total_bytes / 10, + } || resources.container_bytes.unwrap_or(resources.process_bytes) >= process_ceiling; let cpu_pressure = resources.cpu_percent >= 90.0; let automatic = settings.mode == CapacityMode::Auto; let conversations = if automatic { @@ -321,3 +324,51 @@ pub fn value_bytes(value: &Value) -> usize { _ => 32, } } + +#[cfg(test)] +mod memory_pressure_tests { + use super::*; + + const GIB: u64 = 1024 * 1024 * 1024; + + fn resources(available_bytes: u64, os_pressure_level: Option) -> monitor::Resources { + monitor::Resources { + total_bytes: 18 * GIB, + available_bytes, + process_bytes: 0, + container_bytes: None, + cpu_percent: 0.0, + cores: 8, + connection_ceiling: 4096, + os_pressure_level, + } + } + + #[test] + fn the_kernel_pressure_level_beats_a_zero_available_estimate() { + // macOS 的压缩内存会把可用页算成 0;内核说「正常」时就不能再把后台任务全部闸住。 + let limits = effective(&CapacitySettings::default(), resources(0, Some(1))); + assert!(!limits.memory_pressure, "内核报 1(正常)就不该判内存紧张"); + assert!(limits.background > 0, "内存不紧张时后台并发必须放开,否则 models.refresh 会一直 plugin_busy"); + } + + #[test] + fn a_kernel_warning_level_still_stops_background_work() { + // 可用内存再多也听内核的:2(紧张)就该停后台任务。 + let limits = effective(&CapacitySettings::default(), resources(8 * GIB, Some(2))); + assert!(limits.memory_pressure, "内核报 2(紧张)必须判内存紧张"); + assert_eq!(limits.background, 0, "内存紧张时后台并发为 0 是设计,不是缺陷"); + } + + #[test] + fn without_a_kernel_level_the_available_ratio_still_applies() { + // 非 macOS(或 sysctl 取不到)沿用原判据:可用低于总量一成即紧张。 + let low = effective(&CapacitySettings::default(), resources(0, None)); + assert!(low.memory_pressure, "取不到内核等级时,可用内存为 0 必须判紧张"); + assert_eq!(low.background, 0, "旧判据下紧张时后台并发仍为 0"); + + let high = effective(&CapacitySettings::default(), resources(8 * GIB, None)); + assert!(!high.memory_pressure, "可用内存充足时旧判据也不该误报"); + assert!(high.background > 0, "可用内存充足时后台并发要放开"); + } +} diff --git a/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/monitor.rs b/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/monitor.rs index e572f6b56..df121d8b0 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/monitor.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/monitor.rs @@ -14,6 +14,10 @@ pub struct Resources { pub cpu_percent: f32, pub cores: usize, pub connection_ceiling: usize, + /// 内核给出的内存压力等级(macOS:1 正常 / 2 紧张 / 4 危险)。 + /// 有值时以它为准,比按空闲页推算可靠:macOS 的压缩内存会把 + /// 「free + inactive + purgeable − compressor」压到 0,导致常驻误判。 + pub os_pressure_level: Option, } struct Monitor { @@ -28,6 +32,7 @@ fn fallback() -> Resources { process_bytes: 0, container_bytes: None, cpu_percent: 0.0, cores: std::thread::available_parallelism().map(usize::from).unwrap_or(1), connection_ceiling: connection_ceiling(), + os_pressure_level: None, } } @@ -60,13 +65,36 @@ pub fn sample() -> Resources { container_bytes = Some(cgroup.rss); } let resources = Resources { total_bytes, available_bytes, process_bytes, container_bytes, cpu_percent, cores, - connection_ceiling: connection_ceiling() }; + connection_ceiling: connection_ceiling(), os_pressure_level: os_pressure_level() }; monitor.resources = if total_bytes == 0 { fallback() } else { resources }; monitor.updated = Instant::now(); *last.write().unwrap_or_else(|error| error.into_inner()) = monitor.resources; monitor.resources } +/// 读内核的内存压力等级。取不到时返回 None,调用方退回按空闲页推算。 +#[cfg(target_os = "macos")] +fn os_pressure_level() -> Option { + let name = b"kern.memorystatus_vm_pressure_level\0"; + let mut level: libc::c_int = 0; + let mut size = std::mem::size_of::(); + // SAFETY: 传的是本栈上的 c_int 与它的字节长度,名字以 NUL 结尾; + // sysctlbyname 只在这块缓冲里写一个 c_int,长度不符时会自己返回错误而不越界。 + let status = unsafe { + libc::sysctlbyname( + name.as_ptr() as *const libc::c_char, + &mut level as *mut libc::c_int as *mut libc::c_void, + &mut size, + std::ptr::null_mut(), + 0, + ) + }; + (status == 0 && level > 0).then_some(u8::try_from(level).unwrap_or(u8::MAX)) +} + +#[cfg(not(target_os = "macos"))] +fn os_pressure_level() -> Option { None } + #[cfg(unix)] fn connection_ceiling() -> usize { let mut limit = std::mem::MaybeUninit::::uninit(); From 2a393c8424231933e3aa4b93f9db4bba0e05c8f6 Mon Sep 17 00:00:00 2001 From: cculin Date: Fri, 20 Nov 2026 17:26:18 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20macOS=20=E8=AD=A6?= =?UTF-8?q?=E5=91=8A=E7=BA=A7=E5=86=85=E5=AD=98=E5=8E=8B=E5=8A=9B=E6=8A=8A?= =?UTF-8?q?=E5=AF=B9=E8=AF=9D=E4=B8=80=E8=B5=B7=E6=8C=A1=E6=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 现象:Mac 上 OpenChamber 发消息固定等 10 秒后报「运行容量暂时不足,排队等待已结束」(503 gateway_busy),换模型无效。 原因:d9ee142b 把「可用内存 < 总量/10」换成内核压力等级,但阈值取了 2。macOS 的等级是 1 正常 / 2 警告 / 4 危险,2 在正常使用的机器上是常态(实测 18 GiB、 系统空闲 37% 时仍报 2);判紧张后 background/images 被压到 0,queue::allowed 又按 memory_pressure 整类拒绝 Conversation/Request,每个 chat 请求都排队到超时。 做法:只有 4(critical)才算内存紧张;压力只通过 limits 停后台与图片,不再在 allowed() 里整类拒绝对话 —— 每请求内存由 MemoryBudget 单独限额。 --- .../src/server/core/runtime_capacity/mod.rs | 24 ++++++++++--- .../server/core/runtime_capacity/monitor.rs | 3 +- .../src/server/core/runtime_capacity/queue.rs | 35 ++++++++++++++++--- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/mod.rs b/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/mod.rs index 061524ce1..8bc2ea4b3 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/mod.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/mod.rs @@ -79,8 +79,11 @@ fn effective(settings: &CapacitySettings, resources: monitor::Resources) -> Limi else { settings.memory_mib.saturating_mul(MIB).min((resources.total_bytes * 3 / 4) as usize).max(16 * MIB) }; let process_ceiling = resources.total_bytes.saturating_mul(4) / 5; // 有内核压力等级就以它为准:按空闲页推算在 macOS 上会因为压缩内存而长期误判。 + // macOS 给的是 1 正常 / 2 警告 / 4 危险,只有 4 才代表「内存真的不够用」; + // 2 在正常使用的机器上是常态(内核刚开始压缩/换页就会报),拿它判紧张会把 + // 后台并发压到 0、模型清单刷不出来。 let memory_pressure = match resources.os_pressure_level { - Some(level) => level >= 2, + Some(level) => level >= 4, None => resources.available_bytes < resources.total_bytes / 10, } || resources.container_bytes.unwrap_or(resources.process_bytes) >= process_ceiling; let cpu_pressure = resources.cpu_percent >= 90.0; @@ -353,11 +356,22 @@ mod memory_pressure_tests { } #[test] - fn a_kernel_warning_level_still_stops_background_work() { - // 可用内存再多也听内核的:2(紧张)就该停后台任务。 + fn a_kernel_warning_level_is_not_memory_pressure() { + // macOS 的 1/2/4 是 normal / warning / critical;warning(2)只是「开始压缩/换页」, + // 在正常使用的 Mac 上是常态(实测 18 GiB、系统空闲 37% 时仍报 2)。 + // 拿它判紧张会把后台并发压到 0,客户端连模型清单都刷不出来。 let limits = effective(&CapacitySettings::default(), resources(8 * GIB, Some(2))); - assert!(limits.memory_pressure, "内核报 2(紧张)必须判内存紧张"); - assert_eq!(limits.background, 0, "内存紧张时后台并发为 0 是设计,不是缺陷"); + assert!(!limits.memory_pressure, "内核报 2(warning)不代表内存已经不够用"); + assert!(limits.background > 0, "warning 级不该把后台并发压到 0"); + } + + #[test] + fn a_kernel_critical_level_still_stops_background_work() { + // 4 才是 critical:这时才停后台与图片,给前台腾内存。 + let limits = effective(&CapacitySettings::default(), resources(8 * GIB, Some(4))); + assert!(limits.memory_pressure, "内核报 4(critical)必须判内存紧张"); + assert_eq!(limits.background, 0, "内存危险时后台并发为 0"); + assert_eq!(limits.images, 0, "内存危险时图片并发为 0"); } #[test] diff --git a/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/monitor.rs b/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/monitor.rs index df121d8b0..239330e51 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/monitor.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/monitor.rs @@ -14,9 +14,10 @@ pub struct Resources { pub cpu_percent: f32, pub cores: usize, pub connection_ceiling: usize, - /// 内核给出的内存压力等级(macOS:1 正常 / 2 紧张 / 4 危险)。 + /// 内核给出的内存压力等级(macOS:1 正常 / 2 警告 / 4 危险)。 /// 有值时以它为准,比按空闲页推算可靠:macOS 的压缩内存会把 /// 「free + inactive + purgeable − compressor」压到 0,导致常驻误判。 + /// 判「不够用」只看 4:2 在正常使用的机器上是常态,不是危险。 pub os_pressure_level: Option, } diff --git a/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/queue.rs b/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/queue.rs index 48a7e26c3..e2c12be23 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/queue.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/runtime_capacity/queue.rs @@ -71,11 +71,8 @@ impl Drop for Waiting { } fn block_reason(state: &State, class: Class, provider: &str, limits: &Limits) -> Option { - if limits.memory_pressure - && matches!(class, Class::Conversation | Class::Image | Class::Background | Class::Request | Class::ImageRequest) - { - return Some(RejectReason::MemoryPressure); - } + // 内存压力只体现在 limits 里(background / images 被压到 0),不在这里整类拒绝: + // 对话与 chat 请求每笔都由 MemoryBudget 单独限额,宿主紧张时也不该把用户踢下线。 if limits.cpu_pressure && matches!(class, Class::Image | Class::Background | Class::ImageRequest) && limits.for_class(class) == 0 @@ -245,3 +242,31 @@ pub fn record_slot_rejection(class: Class, waited: Duration) -> CapacityRejectio let mut state = manager.state.lock().unwrap_or_else(|error| error.into_inner()); record_rejection(&mut state, RejectReason::PluginSlotLimit, class, waited, &limits, resources) } + +#[cfg(test)] +mod tests { + use super::*; + + fn limits(memory_pressure: bool) -> Limits { + Limits { + conversations: 8, interactive: 8, + // 生产里 effective() 在压力下把这两项压到 0;这里仍给非零, + // 好让用例单独盯住 block_reason() 自己有没有额外拦人。 + background: 4, background_per_provider: 2, images: 2, connections: 16, + memory_bytes: 1024, queue_requests: 8, queue_bytes: 1024, queue_wait_seconds: 1, + cpu_pressure: false, memory_pressure, + } + } + + #[test] + fn memory_pressure_must_not_block_chat_requests() { + let state = State::default(); + let limits = limits(true); + // 对话类请求(/v1/chat/completions 的 Request、插件流式的 Conversation、 + // 非流式的 Interactive)都靠每请求的 MemoryBudget 兜底,不能因为宿主内存 + // 紧张就整类拒绝 —— 那样用户只会看到排队 10 秒后 503。 + assert!(block_reason(&state, Class::Request, "", &limits).is_none(), "内存紧张也不该挡 chat 请求"); + assert!(block_reason(&state, Class::Conversation, "", &limits).is_none(), "内存紧张也不该挡流式会话"); + assert!(block_reason(&state, Class::Interactive, "", &limits).is_none(), "内存紧张也不该挡交互调用"); + } +}