Tolerate invalid UTF-8 at the module string boundary

src/Renderer.zig

@@ -900,7 +900,7 @@ fn flushSpan(self: *Self, env: emacs.Env) !void {
     if (self.span.text.items.len == 0) return;
 
     const span_start = env.cast(usize, env.f("point", .{}));
-    _ = env.f("insert", .{self.span.text.items});
+    _ = env.f("insert", .{env.makeUtf8String(self.span.text.items)});
 
     const span_end = span_start + self.span.char_len;
     self.repainted = if (self.repainted) |r| .{

src/comint_filter.zig

@@ -297,7 +297,7 @@ pub fn feed(self: *Self, env: emacs.Env, data: []const u8) !emacs.Value {
     self.stream.nextSlice(data);
     try h.closePending();
 
-    const text_val = env.makeString(h.text.items);
+    const text_val = env.makeUtf8String(h.text.items);
     const s = &emacs.sym;
 
     for (h.runs.items) |*run| {

src/emacs.zig

@@ -101,7 +101,17 @@ pub const Env = struct {
         return self.raw.make_float.?(self.raw, n);
     }
 
+    /// Invalid UTF-8 keeps its bad bytes as raw-byte chars instead of signalling.
     pub fn makeString(self: Env, str: []const u8) Value {
+        if (!std.unicode.utf8ValidateSlice(str)) {
+            if (self.makeUnibyteString(str)) |unibyte|
+                return self.f("decode-coding-string", .{ unibyte, sym.@"utf-8-unix" });
+        }
+        return self.makeUtf8String(str);
+    }
+
+    /// STR must be valid UTF-8, e.g. text encoded from decoded codepoints.
+    pub fn makeUtf8String(self: Env, str: []const u8) Value {
         return self.raw.make_string.?(self.raw, str.ptr, @intCast(str.len));
     }
 
@@ -190,16 +200,28 @@ pub const Env = struct {
         return self.extractFloat(self.f("float", .{val}));
     }
 
+    /// After a failed `copy_string_contents`: VAL with its raw-byte chars
+    /// encoded back to bytes, or ExtractStringFailed when the failure was not about them.
+    /// The caught signal is debuggable, so `debug-on-error` still shows it.
+    fn rawByteFallback(self: Env, val: Value) !Value {
+        const exit = self.nonLocalExitGet();
+        self.nonLocalExitClear();
+        if (!self.eq(exit.symbol, sym.@"wrong-type-argument")) return error.ExtractStringFailed;
+        return self.f("encode-coding-string", .{ val, sym.@"utf-8-unix" });
+    }
+
     pub fn extractString(self: Env, val: Value, buf: []u8) ![:0]const u8 {
         var len: isize = @intCast(buf.len);
-        if (self.raw.copy_string_contents.?(self.raw, val, buf.ptr, &len)) {
-            // len includes the null terminator
-            return @ptrCast(buf[0..(@as(usize, @intCast(len)) - 1)]);
+        if (!self.raw.copy_string_contents.?(self.raw, val, buf.ptr, &len)) {
+            const encoded = try self.rawByteFallback(val);
+            len = @intCast(buf.len);
+            if (!self.raw.copy_string_contents.?(self.raw, encoded, buf.ptr, &len)) {
+                self.nonLocalExitClear();
+                return error.ExtractStringFailed;
+            }
         }
-        // Clear the non-local exit so callers (e.g. extractStringAlloc
-        // fallback) can make further API calls.
-        self.nonLocalExitClear();
-        return error.ExtractStringFailed;
+        // len includes the null terminator
+        return @ptrCast(buf[0..(@as(usize, @intCast(len)) - 1)]);
     }
 
     pub fn extractStringAlloc(self: Env, alloc: Allocator, val: Value, buf: *?[]u8) ![:0]u8 {
@@ -209,10 +231,14 @@ pub const Env = struct {
         // the module API catches that signal internally, the catcher's type
         // is `CATCHER_ALL_DEBUGGABLE' - so the user's debugger pops up when
         // `debug-on-error' is set, even though we'd recover and retry.
+        var str = val;
         var len: isize = 0;
-        if (!self.raw.copy_string_contents.?(self.raw, val, null, &len)) {
-            self.nonLocalExitClear();
-            return error.ExtractStringFailed;
+        if (!self.raw.copy_string_contents.?(self.raw, str, null, &len)) {
+            str = try self.rawByteFallback(val);
+            if (!self.raw.copy_string_contents.?(self.raw, str, null, &len)) {
+                self.nonLocalExitClear();
+                return error.ExtractStringFailed;
+            }
         }
         const required: usize = @intCast(len);
 
@@ -221,7 +247,7 @@ pub const Env = struct {
             buf.* = try alloc.alloc(u8, required);
         }
 
-        if (!self.raw.copy_string_contents.?(self.raw, val, buf.*.?.ptr, &len)) {
+        if (!self.raw.copy_string_contents.?(self.raw, str, buf.*.?.ptr, &len)) {
             self.nonLocalExitClear();
             return error.ExtractStringFailed;
         }
@@ -451,6 +477,7 @@ const interned_symbols = [_][:0]const u8{
     "cons",
     "current-buffer",
     "dash",
+    "decode-coding-string",
     "default",
     "default-directory",
     "delete-region",
@@ -459,6 +486,7 @@ const interned_symbols = [_][:0]const u8{
     "display-warning",
     "dot",
     "double-line",
+    "encode-coding-string",
     "eobp",
     "eq",
     "erase-buffer",
@@ -562,10 +590,12 @@ const interned_symbols = [_][:0]const u8{
     "symbol-value",
     "t",
     "temporary-file-directory",
+    "utf-8-unix",
     "wave",
     "window-buffer",
     "window-point",
     "window-start",
+    "wrong-type-argument",
 };
 
 fn SymbolCache(comptime symbols: []const [:0]const u8) type {

test/ghostel-comint-test.el

@@ -272,5 +272,13 @@ notes."
                       :foreground)
                      "#00cd00")))))
 
+
+(ert-deftest ghostel-comint-test-raw-byte-chars-pass-through ()
+  "Raw-byte chars in the input decode like any other bytes instead of erroring."
+  :tags '(native)
+  (ghostel-comint-test--with-state state
+    (should (equal (concat "a" (string #xfffd) "b")
+                   (ghostel--comint-filter state (concat "a" (string #x3fffe6) "b"))))))
+
 (provide 'ghostel-comint-test)
 ;;; ghostel-comint-test.el ends here

test/ghostel-links-test.el

@@ -26,6 +26,20 @@
                                                          (get-text-property link-pos 'help-echo)))
                                           (should (keymapp (get-text-property link-pos 'keymap)))))))
 
+(ert-deftest ghostel-test-osc8-invalid-utf8-uri-renders ()
+  "A URI cut mid-character still renders; the bad byte becomes a raw-byte char."
+  :tags '(native)
+  (ghostel-test--with-terminal-buffer (_buf term 5 80 1000)
+    (let ((inhibit-read-only t))
+      (ghostel--write-vt term (concat "\e]8;;file:///tmp/"
+                                      (encode-coding-string "中文" 'utf-8)
+                                      "\346\e\\link\e]8;;\e\\"))
+      (ghostel--redraw term t)
+      (goto-char (point-min))
+      (should (search-forward "link" nil t))
+      (should (equal (concat "file:///tmp/中文" (string #x3fffe6))
+                     (get-text-property (1- (point)) 'help-echo))))))
+
 (ert-deftest ghostel-test-osc8-no-help-echo-outside-link ()
   "Non-link cells do not carry an OSC8 `help-echo' URI."
   :tags '(native)

test/ghostel-terminal-test.el

@@ -640,5 +640,16 @@ state from the wrong buffer after feeding output to the native module."
       ;; `ghostel--cursor-blink-start' installs, cancelling any timer.
       (kill-buffer buf))))
 
+
+(ert-deftest ghostel-test-write-vt-accepts-raw-byte-chars ()
+  "A string holding raw-byte chars reaches the terminal as those bytes."
+  :tags '(native)
+  (ghostel-test--with-terminal-buffer (_buf term 5 80 1000)
+    (let ((inhibit-read-only t))
+      (ghostel--write-vt term (concat "a" (string #x3fffe6) "b\r\n"))
+      (ghostel--redraw term t)
+      (should (string-match-p "\\`a.b$" (buffer-substring-no-properties
+                                        (point-min) (line-end-position)))))))
+
 (provide 'ghostel-terminal-test)
 ;;; ghostel-terminal-test.el ends here