@@ -22,7 +22,9 @@ class ZyLexer : LexerBase() {
2222 this .tokenStart = startOffset
2323 this .tokenEnd = startOffset
2424 this .tokenType = null
25- advance()
25+ if (startOffset < endOffset) {
26+ advance()
27+ }
2628 }
2729
2830 override fun getState (): Int = 0
@@ -55,59 +57,67 @@ class ZyLexer : LexerBase() {
5557 return
5658 }
5759
58- tokenEnd++
59-
60- tokenType = when {
61- currentChar == ' /' && tokenEnd < bufferEnd && buffer!! [tokenEnd] == ' /' -> {
62- // 单行注释 // 的处理
63- while (tokenEnd < bufferEnd && buffer!! [tokenEnd] != ' \n ' ) {
64- tokenEnd++
65- }
66- ZyTokenTypes .COMMENT
67- }
68- currentChar == ' /' && tokenEnd < bufferEnd && buffer!! [tokenEnd] == ' *' -> {
69- // 多行注释 /* */ 的处理
70- tokenEnd++
71- while (tokenEnd < bufferEnd - 1 ) {
72- if (buffer!! [tokenEnd] == ' *' && buffer!! [tokenEnd + 1 ] == ' /' ) {
73- tokenEnd + = 2
74- break
60+ // 简化的token识别逻辑,减少复杂性
61+ when {
62+ currentChar == ' /' && tokenEnd + 1 < bufferEnd -> {
63+ if (buffer!! [tokenEnd + 1 ] == ' /' ) {
64+ // 单行注释
65+ tokenEnd + = 2
66+ while (tokenEnd < bufferEnd && buffer!! [tokenEnd] != ' \n ' ) {
67+ tokenEnd++
7568 }
69+ tokenType = ZyTokenTypes .COMMENT
70+ } else if (buffer!! [tokenEnd + 1 ] == ' *' ) {
71+ // 多行注释
72+ tokenEnd + = 2
73+ while (tokenEnd < bufferEnd - 1 ) {
74+ if (buffer!! [tokenEnd] == ' *' && buffer!! [tokenEnd + 1 ] == ' /' ) {
75+ tokenEnd + = 2
76+ break
77+ }
78+ tokenEnd++
79+ }
80+ tokenType = ZyTokenTypes .COMMENT
81+ } else {
7682 tokenEnd++
83+ tokenType = ZyTokenTypes .OPERATOR
7784 }
78- ZyTokenTypes .COMMENT
7985 }
8086 currentChar == ' "' || currentChar == ' \' ' -> {
81- // 字符串字面量的处理(支持单引号和双引号)
87+ // 字符串字面量
8288 val quote = currentChar
89+ tokenEnd++
8390 while (tokenEnd < bufferEnd && buffer!! [tokenEnd] != quote) {
8491 if (buffer!! [tokenEnd] == ' \\ ' && tokenEnd + 1 < bufferEnd) {
85- tokenEnd + = 2 // 跳过转义字符(如 \n, \t, \" 等)
92+ tokenEnd + = 2
8693 } else {
8794 tokenEnd++
8895 }
8996 }
9097 if (tokenEnd < bufferEnd) tokenEnd++
91- ZyTokenTypes .STRING
98+ tokenType = ZyTokenTypes .STRING
9299 }
93100 currentChar.isDigit() -> {
94- // 数字字面量的处理(整数和浮点数)
101+ // 数字字面量
102+ tokenEnd++
95103 while (tokenEnd < bufferEnd && (buffer!! [tokenEnd].isDigit() || buffer!! [tokenEnd] == ' .' )) {
96104 tokenEnd++
97105 }
98- ZyTokenTypes .NUMBER
106+ tokenType = ZyTokenTypes .NUMBER
99107 }
100108 currentChar.isLetter() || currentChar == ' _' -> {
101- // 标识符或关键字的处理
109+ // 标识符或关键字
110+ tokenEnd++
102111 while (tokenEnd < bufferEnd && (buffer!! [tokenEnd].isLetterOrDigit() || buffer!! [tokenEnd] == ' _' )) {
103112 tokenEnd++
104113 }
105114 val text = buffer!! .subSequence(tokenStart, tokenEnd).toString()
106- if (isKeyword(text)) ZyTokenTypes .KEYWORD else ZyTokenTypes .IDENTIFIER
115+ tokenType = if (isKeyword(text)) ZyTokenTypes .KEYWORD else ZyTokenTypes .IDENTIFIER
107116 }
108117 else -> {
109- // 操作符和其他符号的处理
110- ZyTokenTypes .OPERATOR
118+ // 其他字符
119+ tokenEnd++
120+ tokenType = ZyTokenTypes .OPERATOR
111121 }
112122 }
113123 }
0 commit comments