Skip to content

Commit 9c7f582

Browse files
paldepindCopilot
paldepind
and
Copilot
committed
JS: Migrate the parser to native TypeScript 7.1
Pin the native compiler and adopt its ESM API. Project native AST nodes into the existing JSON protocol and adapt Java conversion to native fields, token kinds, namespaces, and import attributes. Reuse one lazy API instance across files and prepare-files batches, disposing each program after serialization. Close the API on reset, quit, and EOF; rely on process-exit cleanup for fatal errors. Add wrapper, lifecycle, and Java regression coverage, teach the Java test harness to install native runtimes, and regenerate import-attribute expectations. Cover source-phase import bindings with a QL fixture. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8f505060-5341-43f3-b8e9-1b3d60171a2d
1 parent dba7357 commit 9c7f582

15 files changed

Lines changed: 2401 additions & 1115 deletions

File tree

‎javascript/extractor/lib/typescript/package-lock.json‎

Lines changed: 369 additions & 7 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎javascript/extractor/lib/typescript/package.json‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,20 @@
11
{
22
"name": "typescript-parser-wrapper",
33
"private": true,
4+
"type": "module",
5+
"engines": {
6+
"node": ">=16.20.0"
7+
},
48
"dependencies": {
5-
"typescript": "5.9"
9+
"typescript": "7.1.0-beta"
610
},
711
"scripts": {
8-
"build": "tsc --project tsconfig.json",
9-
"check": "tsc --noEmit --project . && tslint --project .",
12+
"build": "node node_modules/typescript/lib/tsc.js --project tsconfig.json",
13+
"check": "node node_modules/typescript/lib/tsc.js --noEmit --project . && tslint --project .",
14+
"test": "npm run build && node --test test/lifecycle.test.mjs test/wrapper.test.mjs",
1015
"lint": "tslint --project .",
1116
"lint-fix": "tslint --project . --fix",
12-
"watch": "tsc -p . -w --sourceMap"
17+
"watch": "node node_modules/typescript/lib/tsc.js -p . -w --sourceMap"
1318
},
1419
"devDependencies": {
1520
"@types/node": "^24.3.1"

‎javascript/extractor/lib/typescript/src/ast_extractor.ts‎

Lines changed: 193 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -1,143 +1,217 @@
1-
import * as ts from "typescript";
2-
3-
/**
4-
* Interface that exposes some internal properties we rely on, as well as
5-
* some properties we add ourselves.
6-
*/
7-
export interface AugmentedSourceFile extends ts.SourceFile {
8-
parseDiagnostics?: any[];
9-
/** Internal property that we expose as a workaround. */
10-
redirectInfo?: object | null;
11-
$tokens?: Token[];
12-
$lineStarts?: ReadonlyArray<number>;
13-
}
1+
import { createRequire } from "node:module";
2+
import { dirname, join } from "node:path";
3+
import { pathToFileURL } from "node:url";
4+
import { SyntaxKind } from "typescript/ast";
5+
import * as ts from "typescript/ast";
6+
import type { Diagnostic } from "typescript/sync";
147

15-
export interface AugmentedNode extends ts.Node {
16-
$pos?: any;
17-
$end?: any;
18-
$declarationKind?: string;
8+
export type JsonValue = string | number | boolean | JsonObject | JsonValue[];
9+
export interface JsonObject {
10+
[property: string]: JsonValue;
1911
}
2012

21-
export type AugmentedPos = number;
13+
/** Child and scalar properties consumed by the Java AST converter. */
14+
const astProperties = [
15+
"argument",
16+
"argumentExpression",
17+
"arguments",
18+
"assertsModifier",
19+
"asteriskToken",
20+
"attributes",
21+
"block",
22+
"body",
23+
"caseBlock",
24+
"catchClause",
25+
"checkType",
26+
"children",
27+
"clauses",
28+
"closingElement",
29+
"closingFragment",
30+
"condition",
31+
"constraint",
32+
"declarationList",
33+
"declarations",
34+
"defaultType",
35+
"dotDotDotToken",
36+
"elements",
37+
"elementType",
38+
"elseStatement",
39+
"exclamationToken",
40+
"exportClause",
41+
"expression",
42+
"exprName",
43+
"extendsType",
44+
"falseType",
45+
"finallyBlock",
46+
"head",
47+
"heritageClauses",
48+
"importClause",
49+
"incrementor",
50+
"indexType",
51+
"initializer",
52+
"keywordToken",
53+
"label",
54+
"left",
55+
"literal",
56+
"members",
57+
"modifiers",
58+
"moduleReference",
59+
"moduleSpecifier",
60+
"name",
61+
"namedBindings",
62+
"objectType",
63+
"openingElement",
64+
"openingFragment",
65+
"operand",
66+
"operator",
67+
"operatorToken",
68+
"parameterName",
69+
"parameters",
70+
"phaseModifier",
71+
"postfixToken",
72+
"properties",
73+
"propertyName",
74+
"qualifier",
75+
"questionDotToken",
76+
"questionToken",
77+
"right",
78+
"statement",
79+
"statements",
80+
"tag",
81+
"tagName",
82+
"template",
83+
"templateSpans",
84+
"text",
85+
"thenStatement",
86+
"token",
87+
"trueType",
88+
"tryBlock",
89+
"type",
90+
"typeArguments",
91+
"typeName",
92+
"typeParameter",
93+
"typeParameters",
94+
"types",
95+
"value",
96+
"variableDeclaration",
97+
"whenFalse",
98+
"whenTrue",
99+
];
22100

23-
export interface Token {
24-
kind: ts.SyntaxKind;
25-
tokenPos: AugmentedPos;
26-
text: string;
27-
}
101+
const require = createRequire(import.meta.url);
102+
const packageRoot = dirname(require.resolve("typescript/package.json"));
28103

29-
let skipWhiteSpace = /(?:\s|\/\/.*|\/\*[^]*?\*\/)*/g;
104+
// NOTE: Importing this file directly from the `typescript` package is a bit
105+
// hacky, but the child properties are not exposed through the public API.
106+
const { childProperties } = await import(
107+
pathToFileURL(join(packageRoot, "dist/api/node/protocol.generated.js")).href);
30108

31-
/**
32-
* Invokes the given callback for every AST node in the given tree.
33-
*/
34-
function forEachNode(ast: ts.Node, callback: (node: ts.Node) => void) {
35-
function visit(node: ts.Node) {
36-
ts.forEachChild(node, visit);
37-
callback(node);
109+
const scalarAstProperties = ["keywordToken", "operator", "phaseModifier", "text", "token"];
110+
const astPropertiesByKind = {} as Record<ts.SyntaxKind, readonly string[]>;
111+
for (const kind of Object.values(SyntaxKind)) {
112+
if (typeof kind === "number" && kind !== SyntaxKind.Count) {
113+
const children = childProperties[kind] ?? [];
114+
astPropertiesByKind[kind] = astProperties.filter(property =>
115+
scalarAstProperties.includes(property) || children.includes(property));
38116
}
39-
visit(ast);
40117
}
41118

42-
export function augmentAst(ast: AugmentedSourceFile, code: string) {
43-
ast.$lineStarts = ast.getLineStarts();
44-
45-
/**
46-
* Converts a numeric offset to the value expected by the Java counterpart of the extractor.
47-
*/
48-
function augmentPos(pos: number, shouldSkipWhitespace?: boolean): AugmentedPos {
49-
// skip over leading spaces/comments
50-
if (shouldSkipWhitespace) {
51-
skipWhiteSpace.lastIndex = pos;
52-
pos += skipWhiteSpace.exec(code)[0].length;
53-
}
54-
return pos;
119+
const skipWhiteSpace = /(?:\s|\/\/.*|\/\*[^]*?\*\/)*/g;
120+
121+
export function extractAst(ast: ts.SourceFile, diagnostics: readonly Diagnostic[]): JsonObject {
122+
const code = ast.text;
123+
// The native parser still builds assertion ASTs, but now diagnoses this formerly accepted syntax.
124+
const parseDiagnostics = diagnostics.filter(diagnostic => diagnostic.code !== 2880);
125+
126+
function augmentPos(pos: number): number {
127+
skipWhiteSpace.lastIndex = pos;
128+
return pos + skipWhiteSpace.exec(code)![0].length;
55129
}
56130

57-
// Find the position of all tokens where the scanner requires parse-tree information.
58-
// At these offsets, a call to `scanner.reScanX` is required.
59-
type RescanEvent = () => void;
60-
let reScanEvents: RescanEvent[] = [];
61-
let reScanEventPos: number[] = [];
62-
let scanner = ts.createScanner(ts.ScriptTarget.ES2015, false, 1, code);
63-
let reScanSlashToken = scanner.reScanSlashToken.bind(scanner);
64-
let reScanTemplateToken = scanner.reScanTemplateToken.bind(scanner);
65-
let reScanGreaterToken = scanner.reScanGreaterToken.bind(scanner);
66-
if (!ast.parseDiagnostics || ast.parseDiagnostics.length === 0) {
67-
forEachNode(ast, node => {
68-
if (ts.isRegularExpressionLiteral(node)) {
69-
reScanEventPos.push(node.getStart(ast, false));
70-
reScanEvents.push(reScanSlashToken);
71-
}
72-
if (ts.isTemplateMiddle(node) || ts.isTemplateTail(node)) {
73-
reScanEventPos.push(node.getStart(ast, false));
74-
reScanEvents.push(reScanTemplateToken);
75-
}
76-
if (ts.isBinaryExpression(node)) {
77-
let operator = node.operatorToken;
78-
switch (operator.kind) {
79-
case ts.SyntaxKind.GreaterThanEqualsToken:
80-
case ts.SyntaxKind.GreaterThanGreaterThanEqualsToken:
81-
case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken:
82-
case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanToken:
83-
case ts.SyntaxKind.GreaterThanGreaterThanToken:
84-
reScanEventPos.push(operator.getStart(ast, false));
85-
reScanEvents.push(reScanGreaterToken);
86-
break;
87-
}
88-
}
89-
});
131+
function projectValue(value: unknown): JsonValue {
132+
if (Array.isArray(value)) return Array.from(value, projectValue);
133+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
134+
return value;
135+
}
136+
if (value != null && typeof value === "object" && "forEachChild" in value) {
137+
return projectNode(value as ts.Node);
138+
}
139+
throw new Error("Unexpected TypeScript AST property value");
90140
}
91141

92-
reScanEventPos.push(Infinity);
142+
function projectNode(node: ts.Node): JsonObject {
143+
const result: JsonObject = {
144+
kind: node.kind,
145+
flags: node.flags,
146+
$pos: augmentPos(node.pos),
147+
$end: node.end,
148+
};
149+
const properties = node as ts.Node & { [property: string]: unknown };
150+
for (const property of astPropertiesByKind[node.kind]) {
151+
const value = properties[property];
152+
if (value != null) result[property] = projectValue(value);
153+
}
93154

94-
// add tokens and comments to the AST
95-
ast.$tokens = [];
96-
let rescanEventIndex = 0;
97-
let nextRescanPosition = reScanEventPos[0];
98-
let tk;
99-
do {
100-
tk = scanner.scan();
101-
if (scanner.getTokenPos() === nextRescanPosition) {
102-
let callback = reScanEvents[rescanEventIndex];
103-
callback();
104-
++rescanEventIndex;
105-
nextRescanPosition = reScanEventPos[rescanEventIndex];
155+
// Native nodes share boolean getters even on kinds where they are not meaningful.
156+
if (ts.isExportAssignment(node)) result.isExportEquals = node.isExportEquals;
157+
if (ts.isExportDeclaration(node) || ts.isImportEqualsDeclaration(node)
158+
|| ts.isImportSpecifier(node) || ts.isExportSpecifier(node)) {
159+
result.isTypeOnly = node.isTypeOnly;
106160
}
107-
ast.$tokens.push({
108-
kind: tk,
109-
tokenPos: augmentPos(scanner.getTokenPos()),
110-
text: scanner.getTokenText(),
111-
});
112-
} while (tk !== ts.SyntaxKind.EndOfFileToken);
161+
if (ts.isImportTypeNode(node)) result.isTypeOf = node.isTypeOf;
113162

114-
if (ast.parseDiagnostics) {
115-
ast.parseDiagnostics.forEach(d => {
116-
delete d.file;
117-
d.$pos = augmentPos(d.start);
118-
});
163+
return result;
119164
}
120165

121-
visitAstNode(ast);
122-
function visitAstNode(node: AugmentedNode) {
123-
ts.forEachChild(node, visitAstNode);
166+
const result = projectNode(ast);
167+
result.$lineStarts = Array.from(ast.getLineStarts());
168+
result.$tokens = extractTokens(ast, parseDiagnostics.length === 0);
169+
result.parseDiagnostics = parseDiagnostics.map(diagnostic => ({
170+
messageText: diagnostic.text,
171+
$pos: diagnostic.pos,
172+
}));
173+
return result;
174+
}
124175

125-
// fill in line/column info
126-
if ("pos" in node) {
127-
node.$pos = augmentPos(node.pos, true);
128-
}
129-
if ("end" in node) {
130-
node.$end = augmentPos(node.end);
131-
}
176+
function extractTokens(ast: ts.SourceFile, rescan: boolean): JsonObject[] {
177+
const scanner = ts.createScanner(false, ts.LanguageVariant.JSX, ast.text);
178+
const rescans = new Map<number, () => ts.SyntaxKind>();
179+
if (rescan) visit(ast);
132180

133-
if (ts.isVariableDeclarationList(node as any)) {
134-
if (node.flags & ts.NodeFlags.Let) {
135-
node.$declarationKind = "let";
136-
} else if (node.flags & ts.NodeFlags.Const) {
137-
node.$declarationKind = "const";
138-
} else {
139-
node.$declarationKind = "var";
181+
function visit(node: ts.Node): void {
182+
node.forEachChild(visit);
183+
if (ts.isRegularExpressionLiteral(node)) {
184+
rescans.set(node.getStart(ast), () => scanner.reScanSlashToken());
185+
} else if (ts.isTemplateMiddle(node) || ts.isTemplateTail(node)) {
186+
const template = node.parent.parent;
187+
const tagged = ts.isTemplateExpression(template) && ts.isTaggedTemplateExpression(template.parent);
188+
rescans.set(node.getStart(ast), () => scanner.reScanTemplateToken(tagged));
189+
} else if (ts.isBinaryExpression(node)) {
190+
const operator = node.operatorToken;
191+
switch (operator.kind) {
192+
case ts.SyntaxKind.GreaterThanEqualsToken:
193+
case ts.SyntaxKind.GreaterThanGreaterThanEqualsToken:
194+
case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken:
195+
case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanToken:
196+
case ts.SyntaxKind.GreaterThanGreaterThanToken:
197+
rescans.set(operator.getStart(ast), () => scanner.reScanGreaterToken());
198+
break;
140199
}
141200
}
142201
}
202+
203+
const tokens: JsonObject[] = [];
204+
let kind: ts.SyntaxKind;
205+
do {
206+
kind = scanner.scan();
207+
const callback = rescans.get(scanner.getTokenStart());
208+
// Keep the pre-rescan kind, as in the original Java-facing token protocol.
209+
if (callback != null) callback();
210+
tokens.push({
211+
kind,
212+
tokenPos: scanner.getTokenStart(),
213+
text: scanner.getTokenText(),
214+
});
215+
} while (kind !== ts.SyntaxKind.EndOfFile);
216+
return tokens;
143217
}

0 commit comments

Comments
 (0)