|
| 1 | +/** |
| 2 | + * Fenced-code highlighting after the highlight.js → tree-sitter swap. |
| 3 | + * |
| 4 | + * Colour lives in the native renderer: a fenced block paints through |
| 5 | + * MarkdownRenderable with transcriptSyntaxStyle() over the bundled |
| 6 | + * tree-sitter grammars (javascript, typescript, markdown, zig). The |
| 7 | + * synchronous StyledSegment model keeps geometry only — plain code segments |
| 8 | + * cached by width. These tests pin both halves: the plain sync fallback and |
| 9 | + * the supported/unsupported language parity the native renderer applies. |
| 10 | + */ |
| 11 | + |
1 | 12 | import { describe, expect, test } from "bun:test"; |
2 | | -import type { StyledSegment } from "./markdown-parser.js"; |
| 13 | +import { MarkdownRenderable, RGBA, type CapturedSpan } from "@opentui/core"; |
| 14 | +import { withTestRenderer, type Harness } from "./harness"; |
| 15 | +import { appendStreamRow } from "./shell/chrome.js"; |
| 16 | +import { createAppShell } from "./shell/index.js"; |
| 17 | +import { transcriptSyntaxStyle } from "./stream"; |
3 | 18 | import { highlightCode } from "./syntax-highlight.js"; |
4 | | -import { color } from "./semantic-theme.js"; |
5 | | - |
6 | | -function segmentFor( |
7 | | - lines: StyledSegment[][], |
8 | | - text: string, |
9 | | -): StyledSegment | undefined { |
10 | | - return lines.flat().find((seg) => seg.text === text); |
11 | | -} |
| 19 | +import { UI } from "./theme"; |
12 | 20 |
|
13 | 21 | describe("highlightCode", () => { |
14 | | - test("colours keywords, strings, and comments from the theme tokens", () => { |
15 | | - const lines = highlightCode('const x = "hi"; // note', "javascript", 80); |
16 | | - expect(segmentFor(lines, "const")?.color).toBe(color("syntaxKeyword")); |
17 | | - expect(segmentFor(lines, '"hi"')?.color).toBe(color("syntaxString")); |
18 | | - expect(segmentFor(lines, "// note")?.color).toBe(color("syntaxComment")); |
19 | | - }); |
20 | | - |
21 | | - test("colours numbers from the syntaxNumber token", () => { |
22 | | - const lines = highlightCode("let n = 42;", "javascript", 80); |
23 | | - expect(segmentFor(lines, "42")?.color).toBe(color("syntaxNumber")); |
| 22 | + test("segments every line as plain code with no colours", () => { |
| 23 | + expect(highlightCode('const x = "hi";\n// yo', "javascript", 80)).toEqual([ |
| 24 | + [{ text: 'const x = "hi";', code: true }], |
| 25 | + [{ text: "// yo", code: true }], |
| 26 | + ]); |
24 | 27 | }); |
25 | 28 |
|
26 | | - test("every highlighted segment is marked as code", () => { |
| 29 | + test("the cache is keyed by width", () => { |
27 | 30 | const lines = highlightCode("const x = 1;", "javascript", 80); |
28 | | - for (const seg of lines.flat()) expect(seg.code).toBe(true); |
| 31 | + expect(highlightCode("const x = 1;", "javascript", 80)).toBe(lines); |
| 32 | + expect(highlightCode("const x = 1;", "javascript", 81)).not.toBe(lines); |
| 33 | + expect(highlightCode("const x = 1;", "javascript", 81)).toEqual(lines); |
29 | 34 | }); |
| 35 | +}); |
30 | 36 |
|
31 | | - test("preserves line count so wrapping stays aligned", () => { |
32 | | - const code = "line one\nline two\nline three"; |
33 | | - expect(highlightCode(code, "javascript", 80)).toHaveLength(3); |
34 | | - }); |
| 37 | +const HEADING_FG = RGBA.fromHex(UI.heading); |
| 38 | +const KEYWORD_FG = RGBA.fromHex(UI.inFlightBright); |
| 39 | +const STRING_FG = RGBA.fromHex(UI.done); |
| 40 | +const COMMENT_FG = RGBA.fromHex(UI.textFaint); |
| 41 | +const FUNCTION_FG = RGBA.fromHex(UI.inFlight); |
| 42 | +const SYNTAX_FGS = [KEYWORD_FG, STRING_FG, COMMENT_FG, FUNCTION_FG]; |
35 | 43 |
|
36 | | - test("decodes HTML entities back to their source characters", () => { |
37 | | - const lines = highlightCode("const ok = a > b && c < d;", "javascript", 80); |
38 | | - const rendered = lines |
39 | | - .flat() |
40 | | - .map((s) => s.text) |
41 | | - .join(""); |
42 | | - expect(rendered).toContain(">"); |
43 | | - expect(rendered).toContain("<"); |
44 | | - expect(rendered).toContain("&&"); |
45 | | - }); |
| 44 | +function renderFence(h: Harness, fence: string): void { |
| 45 | + h.root.add( |
| 46 | + new MarkdownRenderable(h.renderer, { |
| 47 | + syntaxStyle: transcriptSyntaxStyle(), |
| 48 | + fg: UI.text, |
| 49 | + width: 80, |
| 50 | + flexShrink: 0, |
| 51 | + content: fence, |
| 52 | + streaming: false, |
| 53 | + }), |
| 54 | + ); |
| 55 | +} |
46 | 56 |
|
47 | | - test("falls back to uncoloured code for an unknown language", () => { |
48 | | - const lines = highlightCode("some plain text", "not-a-language", 80); |
49 | | - for (const seg of lines.flat()) { |
50 | | - expect(seg.code).toBe(true); |
51 | | - expect(seg.color).toBeUndefined(); |
52 | | - } |
53 | | - }); |
| 57 | +async function settleSpans( |
| 58 | + h: Harness, |
| 59 | + ready: (spans: CapturedSpan[]) => boolean, |
| 60 | +): Promise<CapturedSpan[]> { |
| 61 | + let spans = h.captureSpans().lines.flatMap((line) => line.spans); |
| 62 | + for (let i = 0; i < 400 && !ready(spans); i++) { |
| 63 | + await h.renderOnce(); |
| 64 | + await new Promise((resolve) => setTimeout(resolve, 25)); |
| 65 | + spans = h.captureSpans().lines.flatMap((line) => line.spans); |
| 66 | + } |
| 67 | + return spans; |
| 68 | +} |
54 | 69 |
|
55 | | - test("falls back to uncoloured code when the language is absent", () => { |
56 | | - const lines = highlightCode("plain fenced text", undefined, 80); |
57 | | - expect(segmentFor(lines, "plain fenced text")?.color).toBeUndefined(); |
58 | | - }); |
| 70 | +function coloured(spans: CapturedSpan[], text: string, fg: RGBA): boolean { |
| 71 | + return spans.some((s) => s.text.includes(text) && s.fg.equals(fg)); |
| 72 | +} |
59 | 73 |
|
60 | | - test("caches identical (text, language, width) requests", () => { |
61 | | - const first = highlightCode("const cached = 1;", "javascript", 80); |
62 | | - const second = highlightCode("const cached = 1;", "javascript", 80); |
63 | | - expect(second).toBe(first); |
64 | | - }); |
| 74 | +function hasSyntaxColour(spans: CapturedSpan[], text: string): boolean { |
| 75 | + return spans.some( |
| 76 | + (span) => |
| 77 | + span.text.includes(text) && SYNTAX_FGS.some((fg) => span.fg.equals(fg)), |
| 78 | + ); |
| 79 | +} |
65 | 80 |
|
66 | | - test("keys the cache on width so a resize does not reuse stale output", () => { |
67 | | - const narrow = highlightCode("const w = 1;", "javascript", 40); |
68 | | - const wide = highlightCode("const w = 1;", "javascript", 120); |
69 | | - expect(wide).not.toBe(narrow); |
| 81 | +async function expectTranscriptHighlighting( |
| 82 | + h: Harness, |
| 83 | + supportedLanguage: "js" | "ts", |
| 84 | + streaming: boolean, |
| 85 | +): Promise<void> { |
| 86 | + const shell = createAppShell(h.renderer, { |
| 87 | + terminal: { columns: 80, rows: 40 }, |
| 88 | + wireKeys: false, |
70 | 89 | }); |
| 90 | + try { |
| 91 | + appendStreamRow(shell, { |
| 92 | + role: "assistant", |
| 93 | + streaming, |
| 94 | + text: [ |
| 95 | + `\`\`\`${supportedLanguage}`, |
| 96 | + "const supported_route = 42;", |
| 97 | + "```", |
| 98 | + "", |
| 99 | + "```python", |
| 100 | + "def python_plain():", |
| 101 | + ' return "visible"', |
| 102 | + "```", |
| 103 | + ].join("\n"), |
| 104 | + }); |
| 105 | + const spans = await settleSpans( |
| 106 | + h, |
| 107 | + (current) => |
| 108 | + coloured(current, "const", KEYWORD_FG) && |
| 109 | + current.some((span) => span.text.includes("python_plain")), |
| 110 | + ); |
| 111 | + |
| 112 | + expect(coloured(spans, "const", KEYWORD_FG)).toBe(true); |
| 113 | + expect(h.captureCharFrame()).toContain("python_plain"); |
| 114 | + expect(hasSyntaxColour(spans, "python_plain")).toBe(false); |
| 115 | + } finally { |
| 116 | + shell.dispose(); |
| 117 | + } |
| 118 | +} |
| 119 | + |
| 120 | +describe("production transcript fenced rendering", () => { |
| 121 | + test("settled assistant fences use production syntax ownership", async () => { |
| 122 | + await withTestRenderer( |
| 123 | + (h) => expectTranscriptHighlighting(h, "js", false), |
| 124 | + { width: 80, height: 40 }, |
| 125 | + ); |
| 126 | + }, 30000); |
| 127 | + |
| 128 | + test("streaming assistant fences use production syntax ownership", async () => { |
| 129 | + await withTestRenderer((h) => expectTranscriptHighlighting(h, "ts", true), { |
| 130 | + width: 80, |
| 131 | + height: 40, |
| 132 | + }); |
| 133 | + }, 30000); |
| 134 | +}); |
| 135 | + |
| 136 | +describe("native fenced rendering", () => { |
| 137 | + test("a js fence colours keywords, strings, comments, and numbers", async () => { |
| 138 | + await withTestRenderer( |
| 139 | + async (h) => { |
| 140 | + renderFence( |
| 141 | + h, |
| 142 | + '```js\nconst greeting = "hi"; // yo\nconst n = 42;\n```', |
| 143 | + ); |
| 144 | + const spans = await settleSpans( |
| 145 | + h, |
| 146 | + (s) => |
| 147 | + coloured(s, "const", KEYWORD_FG) && |
| 148 | + coloured(s, "hi", STRING_FG) && |
| 149 | + coloured(s, "// yo", COMMENT_FG) && |
| 150 | + coloured(s, "42", STRING_FG), |
| 151 | + ); |
| 152 | + expect(coloured(spans, "const", KEYWORD_FG)).toBe(true); |
| 153 | + expect(coloured(spans, "hi", STRING_FG)).toBe(true); |
| 154 | + expect(coloured(spans, "// yo", COMMENT_FG)).toBe(true); |
| 155 | + expect(coloured(spans, "42", STRING_FG)).toBe(true); |
| 156 | + }, |
| 157 | + { width: 80, height: 24 }, |
| 158 | + ); |
| 159 | + }, 30000); |
| 160 | + |
| 161 | + test("a ts fence colours keywords and numbers", async () => { |
| 162 | + await withTestRenderer( |
| 163 | + async (h) => { |
| 164 | + renderFence(h, "```ts\nconst n: number = 42;\n```"); |
| 165 | + const spans = await settleSpans( |
| 166 | + h, |
| 167 | + (s) => |
| 168 | + coloured(s, "const", KEYWORD_FG) && coloured(s, "42", STRING_FG), |
| 169 | + ); |
| 170 | + expect(coloured(spans, "const", KEYWORD_FG)).toBe(true); |
| 171 | + expect(coloured(spans, "42", STRING_FG)).toBe(true); |
| 172 | + }, |
| 173 | + { width: 80, height: 24 }, |
| 174 | + ); |
| 175 | + }, 30000); |
| 176 | + |
| 177 | + test("a markdown fence colours headings", async () => { |
| 178 | + await withTestRenderer( |
| 179 | + async (h) => { |
| 180 | + renderFence(h, "```markdown\n# heading\n```"); |
| 181 | + const spans = await settleSpans(h, (s) => |
| 182 | + coloured(s, "heading", HEADING_FG), |
| 183 | + ); |
| 184 | + expect(coloured(spans, "heading", HEADING_FG)).toBe(true); |
| 185 | + }, |
| 186 | + { width: 80, height: 24 }, |
| 187 | + ); |
| 188 | + }, 30000); |
| 189 | + |
| 190 | + test("jsx and tsx fences highlight through their grammar aliases", async () => { |
| 191 | + for (const fence of [ |
| 192 | + "```jsx\nconst el = <div />;\n```", |
| 193 | + "```tsx\nconst el = <div />;\n```", |
| 194 | + ]) { |
| 195 | + await withTestRenderer( |
| 196 | + async (h) => { |
| 197 | + renderFence(h, fence); |
| 198 | + const spans = await settleSpans(h, (s) => |
| 199 | + coloured(s, "const", KEYWORD_FG), |
| 200 | + ); |
| 201 | + expect(coloured(spans, "const", KEYWORD_FG)).toBe(true); |
| 202 | + }, |
| 203 | + { width: 80, height: 24 }, |
| 204 | + ); |
| 205 | + } |
| 206 | + }, 30000); |
| 207 | + |
| 208 | + test("a zig fence colours keywords", async () => { |
| 209 | + await withTestRenderer( |
| 210 | + async (h) => { |
| 211 | + renderFence(h, "```zig\nconst x: i32 = 1;\n```"); |
| 212 | + const spans = await settleSpans(h, (s) => |
| 213 | + coloured(s, "const", KEYWORD_FG), |
| 214 | + ); |
| 215 | + expect(coloured(spans, "const", KEYWORD_FG)).toBe(true); |
| 216 | + }, |
| 217 | + { width: 80, height: 24 }, |
| 218 | + ); |
| 219 | + }, 30000); |
| 220 | + |
| 221 | + test("python, go, and rust fences render PLAIN with no syntax colours", async () => { |
| 222 | + const fences: [string, string][] = [ |
| 223 | + ['```python\ndef greet():\n return "hi" # yo\n```', "def greet"], |
| 224 | + ["```go\npackage main\n\nfunc main() {}\n```", "func main"], |
| 225 | + ['```rust\nfn main() {\n println!("hi");\n}\n```', "fn main"], |
| 226 | + ]; |
| 227 | + for (const [fence, marker] of fences) { |
| 228 | + await withTestRenderer( |
| 229 | + async (h) => { |
| 230 | + renderFence(h, fence); |
| 231 | + const spans = await settleSpans(h, () => |
| 232 | + h.captureCharFrame().includes(marker), |
| 233 | + ); |
| 234 | + expect(h.captureCharFrame()).toContain(marker); |
| 235 | + expect( |
| 236 | + spans.some((s) => SYNTAX_FGS.some((fg) => s.fg.equals(fg))), |
| 237 | + ).toBe(false); |
| 238 | + }, |
| 239 | + { width: 80, height: 24 }, |
| 240 | + ); |
| 241 | + } |
| 242 | + }, 60000); |
71 | 243 | }); |
0 commit comments