feat: 适配 Windows 换行符
- 修改 NewlineTokenScanner 以支持 \r\n 换行符 - 增加对 \r 换行符的处理 - 优化 scanToken 方法,兼容不同操作系统换行符
This commit is contained in:
parent
69b4a418ba
commit
ac5b73e320
@ -11,12 +11,6 @@ import org.jcnc.snow.compiler.lexer.token.TokenType;
|
|||||||
*/
|
*/
|
||||||
public class NewlineTokenScanner extends AbstractTokenScanner {
|
public class NewlineTokenScanner extends AbstractTokenScanner {
|
||||||
|
|
||||||
// 定义状态枚举
|
|
||||||
private enum State {
|
|
||||||
INITIAL,
|
|
||||||
NEWLINE
|
|
||||||
}
|
|
||||||
|
|
||||||
// 当前状态
|
// 当前状态
|
||||||
private State currentState = State.INITIAL;
|
private State currentState = State.INITIAL;
|
||||||
|
|
||||||
@ -31,7 +25,7 @@ public class NewlineTokenScanner extends AbstractTokenScanner {
|
|||||||
@Override
|
@Override
|
||||||
public boolean canHandle(char c, LexerContext ctx) {
|
public boolean canHandle(char c, LexerContext ctx) {
|
||||||
// 只有当处于 INITIAL 状态,并且遇到换行符时,才可以处理
|
// 只有当处于 INITIAL 状态,并且遇到换行符时,才可以处理
|
||||||
return currentState == State.INITIAL && c == '\n';
|
return currentState == State.INITIAL && (c == '\n' || c == '\r');
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -45,16 +39,33 @@ public class NewlineTokenScanner extends AbstractTokenScanner {
|
|||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
protected Token scanToken(LexerContext ctx, int line, int col) {
|
protected Token scanToken(LexerContext ctx, int line, int col) {
|
||||||
// 状态转换为 NEWLINE
|
|
||||||
currentState = State.NEWLINE;
|
currentState = State.NEWLINE;
|
||||||
|
|
||||||
// 执行换行符扫描,生成 token
|
char first = ctx.peek();
|
||||||
|
String lexeme;
|
||||||
|
|
||||||
ctx.advance();
|
ctx.advance();
|
||||||
Token newlineToken = new Token(TokenType.NEWLINE, "\n", line, col);
|
if (first == '\r') {
|
||||||
|
// 检查是否是 \r\n
|
||||||
|
if (!ctx.isAtEnd() && ctx.peek() == '\n') {
|
||||||
|
ctx.advance();
|
||||||
|
lexeme = "\r\n";
|
||||||
|
} else {
|
||||||
|
lexeme = "\r";
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// 一定是 \n
|
||||||
|
lexeme = "\n";
|
||||||
|
}
|
||||||
|
|
||||||
// 扫描完成后,恢复状态为 INITIAL
|
Token newlineToken = new Token(TokenType.NEWLINE, lexeme, line, col);
|
||||||
currentState = State.INITIAL;
|
currentState = State.INITIAL;
|
||||||
|
|
||||||
return newlineToken;
|
return newlineToken;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 定义状态枚举
|
||||||
|
private enum State {
|
||||||
|
INITIAL,
|
||||||
|
NEWLINE
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user