feat: 适配 Windows 换行符

- 修改 NewlineTokenScanner 以支持 \r\n 换行符
- 增加对 \r 换行符的处理
- 优化 scanToken 方法,兼容不同操作系统换行符
This commit is contained in:
Luke 2025-07-17 11:27:03 +08:00
parent 69b4a418ba
commit ac5b73e320

View File

@ -11,12 +11,6 @@ import org.jcnc.snow.compiler.lexer.token.TokenType;
*/ */
public class NewlineTokenScanner extends AbstractTokenScanner { public class NewlineTokenScanner extends AbstractTokenScanner {
// 定义状态枚举
private enum State {
INITIAL,
NEWLINE
}
// 当前状态 // 当前状态
private State currentState = State.INITIAL; private State currentState = State.INITIAL;
@ -31,7 +25,7 @@ public class NewlineTokenScanner extends AbstractTokenScanner {
@Override @Override
public boolean canHandle(char c, LexerContext ctx) { public boolean canHandle(char c, LexerContext ctx) {
// 只有当处于 INITIAL 状态并且遇到换行符时才可以处理 // 只有当处于 INITIAL 状态并且遇到换行符时才可以处理
return currentState == State.INITIAL && c == '\n'; return currentState == State.INITIAL && (c == '\n' || c == '\r');
} }
/** /**
@ -45,16 +39,33 @@ public class NewlineTokenScanner extends AbstractTokenScanner {
*/ */
@Override @Override
protected Token scanToken(LexerContext ctx, int line, int col) { protected Token scanToken(LexerContext ctx, int line, int col) {
// 状态转换为 NEWLINE
currentState = State.NEWLINE; currentState = State.NEWLINE;
// 执行换行符扫描生成 token char first = ctx.peek();
String lexeme;
ctx.advance(); ctx.advance();
Token newlineToken = new Token(TokenType.NEWLINE, "\n", line, col); if (first == '\r') {
// 检查是否是 \r\n
if (!ctx.isAtEnd() && ctx.peek() == '\n') {
ctx.advance();
lexeme = "\r\n";
} else {
lexeme = "\r";
}
} else {
// 一定是 \n
lexeme = "\n";
}
// 扫描完成后恢复状态为 INITIAL Token newlineToken = new Token(TokenType.NEWLINE, lexeme, line, col);
currentState = State.INITIAL; currentState = State.INITIAL;
return newlineToken; return newlineToken;
} }
// 定义状态枚举
private enum State {
INITIAL,
NEWLINE
}
} }