From 4785fb5ba5b7b3a6382acecc99dd53e352363246 Mon Sep 17 00:00:00 2001 From: asto Date: Fri, 28 Aug 2026 13:55:32 +0800 Subject: [PATCH] fix(shell): keep valid UTF-8 prefix on fallback The legacy-fallback arm of ShellStreamDecoder handed the whole pending_utf8 buffer, including its certified UTF-8 prefix, to the system code-page decoder. One stray byte in an otherwise valid chunk therefore re-mojibaked all preceding output on Windows, worst on the synchronous decode path where a fresh decoder sees the complete output. Split on Utf8Error::valid_up_to first: emit the valid prefix as-is and pass only the invalid remainder to the legacy decoder (or from_utf8_lossy when no legacy encoding is configured). This restores the prefix-preserving behavior reviewed in #24 (candidate 654490026) that was dropped when the fix was re-landed in #29 from upstream follow-up 0a85b13ba, and reinstates the regression tests lost with it. Note for the next parent sync: the restored test carries the forkguard_ prefix, so the register's forkguard test inventory count moves from 56 to 57 when this is synced. Signed-off-by: asto --- crates/tui/src/tools/shell/output.rs | 33 +++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/crates/tui/src/tools/shell/output.rs b/crates/tui/src/tools/shell/output.rs index 275358904e..5942252c7c 100644 --- a/crates/tui/src/tools/shell/output.rs +++ b/crates/tui/src/tools/shell/output.rs @@ -60,14 +60,26 @@ impl ShellStreamDecoder { self.pending_utf8.drain(..valid_up_to); decoded } - Err(_) => { + Err(error) => { + // Only the bytes past valid_up_to are genuinely invalid; emit + // the certified UTF-8 prefix as-is so a single stray byte in + // otherwise valid output cannot corrupt the whole chunk. + let valid_up_to = error.valid_up_to(); + let valid_prefix = std::str::from_utf8(&self.pending_utf8[..valid_up_to]) + .expect("Utf8Error::valid_up_to must delimit valid UTF-8"); + decoded.push_str(valid_prefix); + let invalid_and_remaining = &self.pending_utf8[valid_up_to..]; if let Some(encoding) = self.legacy_encoding { let mut decoder = encoding.new_decoder_without_bom_handling(); - decoded.push_str(&decode_legacy_chunk(&mut decoder, &self.pending_utf8, last)); + decoded.push_str(&decode_legacy_chunk( + &mut decoder, + invalid_and_remaining, + last, + )); self.pending_utf8.clear(); self.stream_decoder = Some(decoder); } else { - decoded.push_str(&String::from_utf8_lossy(&self.pending_utf8)); + decoded.push_str(&String::from_utf8_lossy(invalid_and_remaining)); self.pending_utf8.clear(); } self.finished = last; @@ -296,6 +308,21 @@ mod tests { decode_shell_bytes_with_legacy(b"ready \xE4", None, true), "ready \u{FFFD}" ); + assert_eq!( + decode_shell_bytes_with_legacy(b"ready \xE4", Some(encoding_rs::WINDOWS_1252), true), + "ready ä" + ); + } + + #[test] + fn forkguard_shell_valid_utf8_prefix_survives_legacy_fallback() { + let mut bytes = "中文".as_bytes().to_vec(); + bytes.push(0x92); + + assert_eq!( + decode_shell_bytes_with_legacy(&bytes, Some(encoding_rs::WINDOWS_1252), true), + "中文’" + ); } #[test]