Использование штатного optimized-форка antlr и собственного LexerATNS…

…imulator

build.gradle.kts

@@ -31,18 +31,18 @@ tasks.withType<JavaCompile> {
     options.encoding = "UTF-8"
 }
 
-val antlrVersion = "fde6a4930aedcd88725a6f01d7293b1bf2e25140"
-val antlrGroupId = "com.github.nixel2007"
-val antlrArtifactId = "antlr4-optimized"
-val junitVersion = "5.6.0-M1"
+val antlrVersion = "4.7.4"
+val antlrGroupId = "com.tunnelvisionlabs"
+val antlrArtifactId = "antlr4"
+val junitVersion = "5.6.0-RC1"
 
 dependencies {
     compile(antlrGroupId, antlrArtifactId, antlrVersion)
     antlr(antlrGroupId, antlrArtifactId, antlrVersion)
 
     testImplementation("org.junit.jupiter", "junit-jupiter-api", junitVersion)
     testRuntime("org.junit.jupiter", "junit-jupiter-engine", junitVersion)
-    testImplementation("org.assertj", "assertj-core", "3.13.2")
+    testImplementation("org.assertj", "assertj-core", "3.14.0")
 
     // https://mvnrepository.com/artifact/commons-io/commons-io
     compile("commons-io", "commons-io", "2.6")

src/jmh/java/com/github/_1c_syntax/bsl/parser/JMXBSLLexerTest.java

@@ -56,7 +56,7 @@ public void testCharStream()
     throws ClassNotFoundException, InvocationTargetException, InstantiationException, IllegalAccessException {
 
     Class<Lexer> lexerClass = (Class<Lexer>) Class.forName("com.github._1c_syntax.bsl.parser." + lexerClassName);
-    Lexer lexer = (Lexer) lexerClass.getDeclaredConstructors()[0].newInstance((Object) null);
+    Lexer lexer = (Lexer) lexerClass.getDeclaredConstructors()[0].newInstance((Object) null, true);
 
     Tokenizer tokenizer = new Tokenizer(content, lexer);
     tokenizer.getTokens();

src/main/antlr/BSLLexer.g4

@@ -21,11 +21,18 @@
  */
 lexer grammar BSLLexer;
 
+@members {
+public BSLLexer(CharStream input, boolean crAwareCostructor) {
+  super(input);
+  _interp = new CRAwareLexerATNSimulator(this, _ATN);
+  validateInputStream(_ATN, input);
+}
+}
+
 // commons
 fragment DIGIT: [0-9];
 LINE_COMMENT: '//' ~[\r\n]* -> channel(HIDDEN);
-NEWLINE: [\r\n] -> channel(HIDDEN);
-WHITE_SPACE: [ \t\f]+ -> channel(HIDDEN);
+WHITE_SPACE: [ \t\f\r\n]+ -> channel(HIDDEN);
 
 // separators
 DOT: '.' -> pushMode(DOT_MODE);

src/main/java/com/github/_1c_syntax/bsl/parser/CRAwareLexerATNSimulator.java

@@ -0,0 +1,55 @@
+/*
+ * This file is a part of BSL Parser.
+ *
+ * Copyright © 2018-2020
+ * Alexey Sosnoviy <[email protected]>, Nikita Gryzlov <[email protected]>, Sergey Batanov <[email protected]>
+ *
+ * SPDX-License-Identifier: LGPL-3.0-or-later
+ *
+ * BSL Parser is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3.0 of the License, or (at your option) any later version.
+ *
+ * BSL Parser is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with BSL Parser.
+ */
+package com.github._1c_syntax.bsl.parser;
+
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.Lexer;
+import org.antlr.v4.runtime.atn.ATN;
+import org.antlr.v4.runtime.atn.LexerATNSimulator;
+
+public class CRAwareLexerATNSimulator extends LexerATNSimulator {
+  public CRAwareLexerATNSimulator(ATN atn) {
+    super(atn);
+  }
+
+  public CRAwareLexerATNSimulator(Lexer recog, ATN atn) {
+    super(recog, atn);
+  }
+
+  @Override
+  public void consume(CharStream input) {
+    int curChar = input.LA(1);
+    if (curChar == '\n') {
+      line++;
+      charPositionInLine = 0;
+    } else if (curChar == '\r') {
+      int nextChar = input.LA(2);
+      if (nextChar != '\n') {
+        line++;
+        charPositionInLine = 0;
+      }
+    } else {
+      charPositionInLine++;
+    }
+    input.consume();
+  }
+}

src/main/java/com/github/_1c_syntax/bsl/parser/Tokenizer.java

@@ -111,7 +111,7 @@ private CommonTokenStream computeTokenStream() {
     }
 
     if (lexer == null) {
-      lexer = new BSLLexer(input);
+      lexer = new BSLLexer(input, true);
     } else {
       lexer.setInputStream(input);
     }

src/test/java/com/github/_1c_syntax/bsl/parser/BSLLexerTest.java

@@ -55,7 +55,7 @@ private List<Token> getTokens(int mode, String inputString) {
     }
 
     if (lexer == null) {
-      lexer = new BSLLexer(input);
+      lexer = new BSLLexer(input, true);
     } else {
       lexer.setInputStream(input);
     }
@@ -152,11 +152,7 @@ void testBOM() {
   void testCRCR() {
     List<Token> tokens = getTokens(BSLLexer.DEFAULT_MODE, "\r\n\r\r\n");
     assert tokens.get(0).getLine() == 1;
-    assert tokens.get(1).getLine() == 1;
-    assert tokens.get(2).getLine() == 2;
-    assert tokens.get(3).getLine() == 3;
-    assert tokens.get(4).getLine() == 3;
-    assert tokens.get(5).getLine() == 4;
+    assert tokens.get(1).getLine() == 4;
   }
 
   @Test

src/test/java/com/github/_1c_syntax/bsl/parser/BSLParserTest.java

@@ -64,7 +64,7 @@ private void setInput(String inputString, int mode) {
       throw new RuntimeException(e);
     }
 
-    BSLLexer lexer = new BSLLexer(input);
+    BSLLexer lexer = new BSLLexer(input, true);
     lexer.removeErrorListener(ConsoleErrorListener.INSTANCE);
     lexer.mode(mode);