Skip to content

Commit bbc5ec5

Browse files
refactor(tui): use native fenced-code highlighting (#1219)
* refactor(tui): use native fenced-code highlighting * fix(tui): pin native highlighting ownership
1 parent cb3e3e6 commit bbc5ec5

6 files changed

Lines changed: 265 additions & 218 deletions

File tree

‎bun.lock‎

Lines changed: 0 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,6 @@
109109
"@modelcontextprotocol/sdk": "^1.29.0",
110110
"@opentui/core": "0.5.11",
111111
"arktype": "catalog:",
112-
"highlight.js": "^11.11.1",
113112
"isomorphic-git": "catalog:"
114113
},
115114
"devDependencies": {

‎src/tui/markdown-parser.test.ts‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import {
77
withholdIncompleteHeading,
88
type StyledSegment,
99
} from "./markdown-parser.js";
10-
import { color } from "./semantic-theme.js";
1110

1211
function firstLine(text: string): StyledSegment[] {
1312
return parseMarkdown(text)[0] ?? [];
@@ -124,19 +123,19 @@ describe("block elements", () => {
124123
expect(lines[0]?.[0]?.text).toContain("╭");
125124
});
126125

127-
test("highlights a fenced block by its language token", () => {
126+
test("a fenced block keeps its language cap with a plain body", () => {
128127
const lines = parseMarkdown('```js\nconst x = "hi";\n```');
129-
const keyword = lines.flat().find((s) => s.text === "const");
130-
expect(keyword?.code).toBe(true);
131-
expect(keyword?.color).toBe(color("syntaxKeyword"));
132-
const str = lines.flat().find((s) => s.text === '"hi"');
133-
expect(str?.color).toBe(color("syntaxString"));
128+
const body = lines.flat().find((s) => s.text.includes("const x"));
129+
expect(body?.code).toBe(true);
130+
expect(body?.color).toBeUndefined();
131+
expect(lines[0]?.map((s) => s.text).join("")).toContain("js");
134132
});
135133

136-
test("an unclosed streaming fence still highlights its body", () => {
134+
test("an unclosed streaming fence keeps a plain body", () => {
137135
const lines = parseMarkdown("```js\nconst x = 1;");
138-
const keyword = lines.flat().find((s) => s.text === "const");
139-
expect(keyword?.color).toBe(color("syntaxKeyword"));
136+
const body = lines.flat().find((s) => s.text.includes("const x"));
137+
expect(body?.code).toBe(true);
138+
expect(body?.color).toBeUndefined();
140139
});
141140

142141
test("drops a half-typed closing fence from the streaming tail", () => {

‎src/tui/syntax-highlight.test.ts‎

Lines changed: 227 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,71 +1,243 @@
1+
/**
2+
* Fenced-code highlighting after the highlight.js → tree-sitter swap.
3+
*
4+
* Colour lives in the native renderer: a fenced block paints through
5+
* MarkdownRenderable with transcriptSyntaxStyle() over the bundled
6+
* tree-sitter grammars (javascript, typescript, markdown, zig). The
7+
* synchronous StyledSegment model keeps geometry only — plain code segments
8+
* cached by width. These tests pin both halves: the plain sync fallback and
9+
* the supported/unsupported language parity the native renderer applies.
10+
*/
11+
112
import { describe, expect, test } from "bun:test";
2-
import type { StyledSegment } from "./markdown-parser.js";
13+
import { MarkdownRenderable, RGBA, type CapturedSpan } from "@opentui/core";
14+
import { withTestRenderer, type Harness } from "./harness";
15+
import { appendStreamRow } from "./shell/chrome.js";
16+
import { createAppShell } from "./shell/index.js";
17+
import { transcriptSyntaxStyle } from "./stream";
318
import { highlightCode } from "./syntax-highlight.js";
4-
import { color } from "./semantic-theme.js";
5-
6-
function segmentFor(
7-
lines: StyledSegment[][],
8-
text: string,
9-
): StyledSegment | undefined {
10-
return lines.flat().find((seg) => seg.text === text);
11-
}
19+
import { UI } from "./theme";
1220

1321
describe("highlightCode", () => {
14-
test("colours keywords, strings, and comments from the theme tokens", () => {
15-
const lines = highlightCode('const x = "hi"; // note', "javascript", 80);
16-
expect(segmentFor(lines, "const")?.color).toBe(color("syntaxKeyword"));
17-
expect(segmentFor(lines, '"hi"')?.color).toBe(color("syntaxString"));
18-
expect(segmentFor(lines, "// note")?.color).toBe(color("syntaxComment"));
19-
});
20-
21-
test("colours numbers from the syntaxNumber token", () => {
22-
const lines = highlightCode("let n = 42;", "javascript", 80);
23-
expect(segmentFor(lines, "42")?.color).toBe(color("syntaxNumber"));
22+
test("segments every line as plain code with no colours", () => {
23+
expect(highlightCode('const x = "hi";\n// yo', "javascript", 80)).toEqual([
24+
[{ text: 'const x = "hi";', code: true }],
25+
[{ text: "// yo", code: true }],
26+
]);
2427
});
2528

26-
test("every highlighted segment is marked as code", () => {
29+
test("the cache is keyed by width", () => {
2730
const lines = highlightCode("const x = 1;", "javascript", 80);
28-
for (const seg of lines.flat()) expect(seg.code).toBe(true);
31+
expect(highlightCode("const x = 1;", "javascript", 80)).toBe(lines);
32+
expect(highlightCode("const x = 1;", "javascript", 81)).not.toBe(lines);
33+
expect(highlightCode("const x = 1;", "javascript", 81)).toEqual(lines);
2934
});
35+
});
3036

31-
test("preserves line count so wrapping stays aligned", () => {
32-
const code = "line one\nline two\nline three";
33-
expect(highlightCode(code, "javascript", 80)).toHaveLength(3);
34-
});
37+
const HEADING_FG = RGBA.fromHex(UI.heading);
38+
const KEYWORD_FG = RGBA.fromHex(UI.inFlightBright);
39+
const STRING_FG = RGBA.fromHex(UI.done);
40+
const COMMENT_FG = RGBA.fromHex(UI.textFaint);
41+
const FUNCTION_FG = RGBA.fromHex(UI.inFlight);
42+
const SYNTAX_FGS = [KEYWORD_FG, STRING_FG, COMMENT_FG, FUNCTION_FG];
3543

36-
test("decodes HTML entities back to their source characters", () => {
37-
const lines = highlightCode("const ok = a > b && c < d;", "javascript", 80);
38-
const rendered = lines
39-
.flat()
40-
.map((s) => s.text)
41-
.join("");
42-
expect(rendered).toContain(">");
43-
expect(rendered).toContain("<");
44-
expect(rendered).toContain("&&");
45-
});
44+
function renderFence(h: Harness, fence: string): void {
45+
h.root.add(
46+
new MarkdownRenderable(h.renderer, {
47+
syntaxStyle: transcriptSyntaxStyle(),
48+
fg: UI.text,
49+
width: 80,
50+
flexShrink: 0,
51+
content: fence,
52+
streaming: false,
53+
}),
54+
);
55+
}
4656

47-
test("falls back to uncoloured code for an unknown language", () => {
48-
const lines = highlightCode("some plain text", "not-a-language", 80);
49-
for (const seg of lines.flat()) {
50-
expect(seg.code).toBe(true);
51-
expect(seg.color).toBeUndefined();
52-
}
53-
});
57+
async function settleSpans(
58+
h: Harness,
59+
ready: (spans: CapturedSpan[]) => boolean,
60+
): Promise<CapturedSpan[]> {
61+
let spans = h.captureSpans().lines.flatMap((line) => line.spans);
62+
for (let i = 0; i < 400 && !ready(spans); i++) {
63+
await h.renderOnce();
64+
await new Promise((resolve) => setTimeout(resolve, 25));
65+
spans = h.captureSpans().lines.flatMap((line) => line.spans);
66+
}
67+
return spans;
68+
}
5469

55-
test("falls back to uncoloured code when the language is absent", () => {
56-
const lines = highlightCode("plain fenced text", undefined, 80);
57-
expect(segmentFor(lines, "plain fenced text")?.color).toBeUndefined();
58-
});
70+
function coloured(spans: CapturedSpan[], text: string, fg: RGBA): boolean {
71+
return spans.some((s) => s.text.includes(text) && s.fg.equals(fg));
72+
}
5973

60-
test("caches identical (text, language, width) requests", () => {
61-
const first = highlightCode("const cached = 1;", "javascript", 80);
62-
const second = highlightCode("const cached = 1;", "javascript", 80);
63-
expect(second).toBe(first);
64-
});
74+
function hasSyntaxColour(spans: CapturedSpan[], text: string): boolean {
75+
return spans.some(
76+
(span) =>
77+
span.text.includes(text) && SYNTAX_FGS.some((fg) => span.fg.equals(fg)),
78+
);
79+
}
6580

66-
test("keys the cache on width so a resize does not reuse stale output", () => {
67-
const narrow = highlightCode("const w = 1;", "javascript", 40);
68-
const wide = highlightCode("const w = 1;", "javascript", 120);
69-
expect(wide).not.toBe(narrow);
81+
async function expectTranscriptHighlighting(
82+
h: Harness,
83+
supportedLanguage: "js" | "ts",
84+
streaming: boolean,
85+
): Promise<void> {
86+
const shell = createAppShell(h.renderer, {
87+
terminal: { columns: 80, rows: 40 },
88+
wireKeys: false,
7089
});
90+
try {
91+
appendStreamRow(shell, {
92+
role: "assistant",
93+
streaming,
94+
text: [
95+
`\`\`\`${supportedLanguage}`,
96+
"const supported_route = 42;",
97+
"```",
98+
"",
99+
"```python",
100+
"def python_plain():",
101+
' return "visible"',
102+
"```",
103+
].join("\n"),
104+
});
105+
const spans = await settleSpans(
106+
h,
107+
(current) =>
108+
coloured(current, "const", KEYWORD_FG) &&
109+
current.some((span) => span.text.includes("python_plain")),
110+
);
111+
112+
expect(coloured(spans, "const", KEYWORD_FG)).toBe(true);
113+
expect(h.captureCharFrame()).toContain("python_plain");
114+
expect(hasSyntaxColour(spans, "python_plain")).toBe(false);
115+
} finally {
116+
shell.dispose();
117+
}
118+
}
119+
120+
describe("production transcript fenced rendering", () => {
121+
test("settled assistant fences use production syntax ownership", async () => {
122+
await withTestRenderer(
123+
(h) => expectTranscriptHighlighting(h, "js", false),
124+
{ width: 80, height: 40 },
125+
);
126+
}, 30000);
127+
128+
test("streaming assistant fences use production syntax ownership", async () => {
129+
await withTestRenderer((h) => expectTranscriptHighlighting(h, "ts", true), {
130+
width: 80,
131+
height: 40,
132+
});
133+
}, 30000);
134+
});
135+
136+
describe("native fenced rendering", () => {
137+
test("a js fence colours keywords, strings, comments, and numbers", async () => {
138+
await withTestRenderer(
139+
async (h) => {
140+
renderFence(
141+
h,
142+
'```js\nconst greeting = "hi"; // yo\nconst n = 42;\n```',
143+
);
144+
const spans = await settleSpans(
145+
h,
146+
(s) =>
147+
coloured(s, "const", KEYWORD_FG) &&
148+
coloured(s, "hi", STRING_FG) &&
149+
coloured(s, "// yo", COMMENT_FG) &&
150+
coloured(s, "42", STRING_FG),
151+
);
152+
expect(coloured(spans, "const", KEYWORD_FG)).toBe(true);
153+
expect(coloured(spans, "hi", STRING_FG)).toBe(true);
154+
expect(coloured(spans, "// yo", COMMENT_FG)).toBe(true);
155+
expect(coloured(spans, "42", STRING_FG)).toBe(true);
156+
},
157+
{ width: 80, height: 24 },
158+
);
159+
}, 30000);
160+
161+
test("a ts fence colours keywords and numbers", async () => {
162+
await withTestRenderer(
163+
async (h) => {
164+
renderFence(h, "```ts\nconst n: number = 42;\n```");
165+
const spans = await settleSpans(
166+
h,
167+
(s) =>
168+
coloured(s, "const", KEYWORD_FG) && coloured(s, "42", STRING_FG),
169+
);
170+
expect(coloured(spans, "const", KEYWORD_FG)).toBe(true);
171+
expect(coloured(spans, "42", STRING_FG)).toBe(true);
172+
},
173+
{ width: 80, height: 24 },
174+
);
175+
}, 30000);
176+
177+
test("a markdown fence colours headings", async () => {
178+
await withTestRenderer(
179+
async (h) => {
180+
renderFence(h, "```markdown\n# heading\n```");
181+
const spans = await settleSpans(h, (s) =>
182+
coloured(s, "heading", HEADING_FG),
183+
);
184+
expect(coloured(spans, "heading", HEADING_FG)).toBe(true);
185+
},
186+
{ width: 80, height: 24 },
187+
);
188+
}, 30000);
189+
190+
test("jsx and tsx fences highlight through their grammar aliases", async () => {
191+
for (const fence of [
192+
"```jsx\nconst el = <div />;\n```",
193+
"```tsx\nconst el = <div />;\n```",
194+
]) {
195+
await withTestRenderer(
196+
async (h) => {
197+
renderFence(h, fence);
198+
const spans = await settleSpans(h, (s) =>
199+
coloured(s, "const", KEYWORD_FG),
200+
);
201+
expect(coloured(spans, "const", KEYWORD_FG)).toBe(true);
202+
},
203+
{ width: 80, height: 24 },
204+
);
205+
}
206+
}, 30000);
207+
208+
test("a zig fence colours keywords", async () => {
209+
await withTestRenderer(
210+
async (h) => {
211+
renderFence(h, "```zig\nconst x: i32 = 1;\n```");
212+
const spans = await settleSpans(h, (s) =>
213+
coloured(s, "const", KEYWORD_FG),
214+
);
215+
expect(coloured(spans, "const", KEYWORD_FG)).toBe(true);
216+
},
217+
{ width: 80, height: 24 },
218+
);
219+
}, 30000);
220+
221+
test("python, go, and rust fences render PLAIN with no syntax colours", async () => {
222+
const fences: [string, string][] = [
223+
['```python\ndef greet():\n return "hi" # yo\n```', "def greet"],
224+
["```go\npackage main\n\nfunc main() {}\n```", "func main"],
225+
['```rust\nfn main() {\n println!("hi");\n}\n```', "fn main"],
226+
];
227+
for (const [fence, marker] of fences) {
228+
await withTestRenderer(
229+
async (h) => {
230+
renderFence(h, fence);
231+
const spans = await settleSpans(h, () =>
232+
h.captureCharFrame().includes(marker),
233+
);
234+
expect(h.captureCharFrame()).toContain(marker);
235+
expect(
236+
spans.some((s) => SYNTAX_FGS.some((fg) => s.fg.equals(fg))),
237+
).toBe(false);
238+
},
239+
{ width: 80, height: 24 },
240+
);
241+
}
242+
}, 60000);
71243
});

0 commit comments

Comments
 (0)