From 9e9f634259eba883e80620619232e3a9b3d726db Mon Sep 17 00:00:00 2001 From: Daniel Kraus Date: Sun, 1 Nov 2026 09:49:40 +0100 Subject: [PATCH 1/2] Tolerate invalid UTF-8 at the module string boundary An OSC 8 URI that is not valid UTF-8, for example a CJK path cut mid-character, made every redraw signal wrong-type-argument utf-8-string-p: the renderer handed the raw bytes to make_string, which rejects them. The same path was open for OSC 7 URLs, OSC 52 payloads, notifications and the comint filter's links. makeString now validates first and decodes invalid input through make_unibyte_string and decode-coding-string, so bad bytes become raw-byte chars instead of aborting the redraw. extractString and extractStringAlloc gain the reverse fallback: when copy_string_contents rejects a multibyte string holding raw-byte chars, re-encode it and retry once, so such strings can be written back to the terminal. Closes #775. --- src/Renderer.zig | 2 +- src/comint_filter.zig | 2 +- src/emacs.zig | 50 +++++++++++++++++++++++++++-------- test/ghostel-comint-test.el | 8 ++++++ test/ghostel-links-test.el | 14 ++++++++++ test/ghostel-terminal-test.el | 11 ++++++++ 6 files changed, 74 insertions(+), 13 deletions(-) diff --git a/src/Renderer.zig b/src/Renderer.zig index b4c01dee..35f8d823 100644 --- a/src/Renderer.zig +++ b/src/Renderer.zig @@ -900,7 +900,7 @@ fn flushSpan(self: *Self, env: emacs.Env) !void { if (self.span.text.items.len == 0) return; const span_start = env.cast(usize, env.f("point", .{})); - _ = env.f("insert", .{self.span.text.items}); + _ = env.f("insert", .{env.makeUtf8String(self.span.text.items)}); const span_end = span_start + self.span.char_len; self.repainted = if (self.repainted) |r| .{ diff --git a/src/comint_filter.zig b/src/comint_filter.zig index f77545fe..ece4cd9d 100644 --- a/src/comint_filter.zig +++ b/src/comint_filter.zig @@ -297,7 +297,7 @@ pub fn feed(self: *Self, env: emacs.Env, data: []const u8) !emacs.Value { self.stream.nextSlice(data); try h.closePending(); - const text_val = env.makeString(h.text.items); + const text_val = env.makeUtf8String(h.text.items); const s = &emacs.sym; for (h.runs.items) |*run| { diff --git a/src/emacs.zig b/src/emacs.zig index 25045de2..45662ee6 100644 --- a/src/emacs.zig +++ b/src/emacs.zig @@ -101,7 +101,15 @@ pub const Env = struct { return self.raw.make_float.?(self.raw, n); } + /// Invalid UTF-8 keeps its bad bytes as raw-byte chars instead of signalling. pub fn makeString(self: Env, str: []const u8) Value { + if (!std.unicode.utf8ValidateSlice(str)) if (self.makeUnibyteString(str)) |unibyte| + return self.f("decode-coding-string", .{ unibyte, sym.@"utf-8-unix" }); + return self.makeUtf8String(str); + } + + /// STR must be valid UTF-8, e.g. text encoded from decoded codepoints. + pub fn makeUtf8String(self: Env, str: []const u8) Value { return self.raw.make_string.?(self.raw, str.ptr, @intCast(str.len)); } @@ -190,16 +198,28 @@ pub const Env = struct { return self.extractFloat(self.f("float", .{val})); } + /// After a failed `copy_string_contents`: VAL with its raw-byte chars + /// encoded back to bytes, or null when the failure was not about them. + /// The caught signal is debuggable, so `debug-on-error` still shows it. + fn rawByteFallback(self: Env, val: Value) ?Value { + const exit = self.nonLocalExitGet(); + self.nonLocalExitClear(); + if (!self.eq(exit.symbol, sym.@"wrong-type-argument")) return null; + return self.f("encode-coding-string", .{ val, sym.@"utf-8-unix" }); + } + pub fn extractString(self: Env, val: Value, buf: []u8) ![:0]const u8 { var len: isize = @intCast(buf.len); - if (self.raw.copy_string_contents.?(self.raw, val, buf.ptr, &len)) { - // len includes the null terminator - return @ptrCast(buf[0..(@as(usize, @intCast(len)) - 1)]); + if (!self.raw.copy_string_contents.?(self.raw, val, buf.ptr, &len)) { + const encoded = self.rawByteFallback(val) orelse return error.ExtractStringFailed; + len = @intCast(buf.len); + if (!self.raw.copy_string_contents.?(self.raw, encoded, buf.ptr, &len)) { + self.nonLocalExitClear(); + return error.ExtractStringFailed; + } } - // Clear the non-local exit so callers (e.g. extractStringAlloc - // fallback) can make further API calls. - self.nonLocalExitClear(); - return error.ExtractStringFailed; + // len includes the null terminator + return @ptrCast(buf[0..(@as(usize, @intCast(len)) - 1)]); } pub fn extractStringAlloc(self: Env, alloc: Allocator, val: Value, buf: *?[]u8) ![:0]u8 { @@ -209,10 +229,14 @@ pub const Env = struct { // the module API catches that signal internally, the catcher's type // is `CATCHER_ALL_DEBUGGABLE' - so the user's debugger pops up when // `debug-on-error' is set, even though we'd recover and retry. + var str = val; var len: isize = 0; - if (!self.raw.copy_string_contents.?(self.raw, val, null, &len)) { - self.nonLocalExitClear(); - return error.ExtractStringFailed; + if (!self.raw.copy_string_contents.?(self.raw, str, null, &len)) { + str = self.rawByteFallback(val) orelse return error.ExtractStringFailed; + if (!self.raw.copy_string_contents.?(self.raw, str, null, &len)) { + self.nonLocalExitClear(); + return error.ExtractStringFailed; + } } const required: usize = @intCast(len); @@ -221,7 +245,7 @@ pub const Env = struct { buf.* = try alloc.alloc(u8, required); } - if (!self.raw.copy_string_contents.?(self.raw, val, buf.*.?.ptr, &len)) { + if (!self.raw.copy_string_contents.?(self.raw, str, buf.*.?.ptr, &len)) { self.nonLocalExitClear(); return error.ExtractStringFailed; } @@ -451,6 +475,7 @@ const interned_symbols = [_][:0]const u8{ "cons", "current-buffer", "dash", + "decode-coding-string", "default", "default-directory", "delete-region", @@ -459,6 +484,7 @@ const interned_symbols = [_][:0]const u8{ "display-warning", "dot", "double-line", + "encode-coding-string", "eobp", "eq", "erase-buffer", @@ -562,10 +588,12 @@ const interned_symbols = [_][:0]const u8{ "symbol-value", "t", "temporary-file-directory", + "utf-8-unix", "wave", "window-buffer", "window-point", "window-start", + "wrong-type-argument", }; fn SymbolCache(comptime symbols: []const [:0]const u8) type { diff --git a/test/ghostel-comint-test.el b/test/ghostel-comint-test.el index 30837941..bbf12553 100644 --- a/test/ghostel-comint-test.el +++ b/test/ghostel-comint-test.el @@ -272,5 +272,13 @@ notes." :foreground) "#00cd00"))))) + +(ert-deftest ghostel-comint-test-raw-byte-chars-pass-through () + "Raw-byte chars in the input decode like any other bytes instead of erroring." + :tags '(native) + (ghostel-comint-test--with-state state + (should (equal (concat "a" (string #xfffd) "b") + (ghostel--comint-filter state (concat "a" (string #x3fffe6) "b")))))) + (provide 'ghostel-comint-test) ;;; ghostel-comint-test.el ends here diff --git a/test/ghostel-links-test.el b/test/ghostel-links-test.el index 9d6aacd4..e2a6aa28 100644 --- a/test/ghostel-links-test.el +++ b/test/ghostel-links-test.el @@ -26,6 +26,20 @@ (get-text-property link-pos 'help-echo))) (should (keymapp (get-text-property link-pos 'keymap))))))) +(ert-deftest ghostel-test-osc8-invalid-utf8-uri-renders () + "A URI cut mid-character still renders; the bad byte becomes a raw-byte char." + :tags '(native) + (ghostel-test--with-terminal-buffer (_buf term 5 80 1000) + (let ((inhibit-read-only t)) + (ghostel--write-vt term (concat "\e]8;;file:///tmp/" + (encode-coding-string "中文" 'utf-8) + "\346\e\\link\e]8;;\e\\")) + (ghostel--redraw term t) + (goto-char (point-min)) + (should (search-forward "link" nil t)) + (should (equal (concat "file:///tmp/中文" (string #x3fffe6)) + (get-text-property (1- (point)) 'help-echo)))))) + (ert-deftest ghostel-test-osc8-no-help-echo-outside-link () "Non-link cells do not carry an OSC8 `help-echo' URI." :tags '(native) diff --git a/test/ghostel-terminal-test.el b/test/ghostel-terminal-test.el index 3d17a82b..22cb2a02 100644 --- a/test/ghostel-terminal-test.el +++ b/test/ghostel-terminal-test.el @@ -640,5 +640,16 @@ state from the wrong buffer after feeding output to the native module." ;; `ghostel--cursor-blink-start' installs, cancelling any timer. (kill-buffer buf)))) + +(ert-deftest ghostel-test-write-vt-accepts-raw-byte-chars () + "A string holding raw-byte chars reaches the terminal as those bytes." + :tags '(native) + (ghostel-test--with-terminal-buffer (_buf term 5 80 1000) + (let ((inhibit-read-only t)) + (ghostel--write-vt term (concat "a" (string #x3fffe6) "b\r\n")) + (ghostel--redraw term t) + (should (string-match-p "\\`a.b$" (buffer-substring-no-properties + (point-min) (line-end-position))))))) + (provide 'ghostel-terminal-test) ;;; ghostel-terminal-test.el ends here From cf68a5367984174434b66dc71f8d0a513b8d6644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Sahl=C3=A9n?= Date: Sun, 1 Nov 2026 22:10:44 +0100 Subject: [PATCH 2/2] Simplify raw-byte fallback error handling --- src/emacs.zig | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/emacs.zig b/src/emacs.zig index 45662ee6..e78d0f3f 100644 --- a/src/emacs.zig +++ b/src/emacs.zig @@ -103,8 +103,10 @@ pub const Env = struct { /// Invalid UTF-8 keeps its bad bytes as raw-byte chars instead of signalling. pub fn makeString(self: Env, str: []const u8) Value { - if (!std.unicode.utf8ValidateSlice(str)) if (self.makeUnibyteString(str)) |unibyte| - return self.f("decode-coding-string", .{ unibyte, sym.@"utf-8-unix" }); + if (!std.unicode.utf8ValidateSlice(str)) { + if (self.makeUnibyteString(str)) |unibyte| + return self.f("decode-coding-string", .{ unibyte, sym.@"utf-8-unix" }); + } return self.makeUtf8String(str); } @@ -199,19 +201,19 @@ pub const Env = struct { } /// After a failed `copy_string_contents`: VAL with its raw-byte chars - /// encoded back to bytes, or null when the failure was not about them. + /// encoded back to bytes, or ExtractStringFailed when the failure was not about them. /// The caught signal is debuggable, so `debug-on-error` still shows it. - fn rawByteFallback(self: Env, val: Value) ?Value { + fn rawByteFallback(self: Env, val: Value) !Value { const exit = self.nonLocalExitGet(); self.nonLocalExitClear(); - if (!self.eq(exit.symbol, sym.@"wrong-type-argument")) return null; + if (!self.eq(exit.symbol, sym.@"wrong-type-argument")) return error.ExtractStringFailed; return self.f("encode-coding-string", .{ val, sym.@"utf-8-unix" }); } pub fn extractString(self: Env, val: Value, buf: []u8) ![:0]const u8 { var len: isize = @intCast(buf.len); if (!self.raw.copy_string_contents.?(self.raw, val, buf.ptr, &len)) { - const encoded = self.rawByteFallback(val) orelse return error.ExtractStringFailed; + const encoded = try self.rawByteFallback(val); len = @intCast(buf.len); if (!self.raw.copy_string_contents.?(self.raw, encoded, buf.ptr, &len)) { self.nonLocalExitClear(); @@ -232,7 +234,7 @@ pub const Env = struct { var str = val; var len: isize = 0; if (!self.raw.copy_string_contents.?(self.raw, str, null, &len)) { - str = self.rawByteFallback(val) orelse return error.ExtractStringFailed; + str = try self.rawByteFallback(val); if (!self.raw.copy_string_contents.?(self.raw, str, null, &len)) { self.nonLocalExitClear(); return error.ExtractStringFailed;