You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@groovy.apache.org by su...@apache.org on 2026/09/12 03:31:43 UTC
(groovy) 01/04: GROOVY-12398: Improve lexer and parser performance
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch GROOVY-12398
in repository https://gitbox.apache.org/repos/asf/groovy.git
commit ceb21868cd7e710afeb8319d003e232444fd3735
Author: Daniel Sun <su...@apache.org>
AuthorDate: Fri Sep 11 04:09:37 2026 +0900
GROOVY-12398: Improve lexer and parser performance
Flatten number-literal fragments to character classes, split ASCII
identifiers onto a predicate-free DFA path, replace regex-based lexer
predicates with direct character tests, and gate annotation element
values so AdaptivePredict does not also explore assignment expressions.
---
src/antlr/GroovyLexer.g4 | 149 ++++++---------------
src/antlr/GroovyParser.g4 | 9 +-
.../groovy/parser/antlr4/SemanticPredicates.java | 78 +++++++----
.../parser/antlr4/SemanticPredicatesTest.groovy | 49 +++++++
.../groovy/parser/antlr4/AbstractLexerTest.java | 64 +++++++++
5 files changed, 214 insertions(+), 135 deletions(-)
diff --git a/src/antlr/GroovyLexer.g4 b/src/antlr/GroovyLexer.g4
index e935a97ae1..252195574c 100644
--- a/src/antlr/GroovyLexer.g4
+++ b/src/antlr/GroovyLexer.g4
@@ -28,8 +28,8 @@
*/
/**
- * The Groovy grammar is based on the official grammar for Java:
- * https://github.com/antlr/grammars-v4/blob/master/java/Java.g4
+ * The Groovy grammar is based on the optimized Java grammar:
+ * https://github.com/antlr/grammars-v4/tree/master/java/java
*/
lexer grammar GroovyLexer;
@@ -48,7 +48,6 @@ options {
private boolean errorIgnored;
private long tokenIndex;
private int lastTokenType;
- private int invalidDigitCount;
/**
* When {@code false}, the {@code val} keyword is treated as a regular
@@ -343,7 +342,7 @@ GStringIdentifier
mode GSTRING_PATH_MODE;
GStringPathPart
- : Dot IdentifierInGString
+ : '.' IdentifierInGString
;
RollBackOne
: . {
@@ -509,6 +508,10 @@ WHILE : 'while';
YIELD : 'yield';
// §3.10.1 Integer Literals
+// Digit sequences use character classes (optimized Java grammar style) so the
+// lexer DFA does not walk a chain of one-character fragment rules. The invalid
+// octal alt validates once at the end rather than counting digits in an action
+// loop, which would prevent DFA compilation of this token.
IntegerLiteral
: ( DecimalIntegerLiteral
@@ -518,12 +521,7 @@ IntegerLiteral
) (Underscore { require(errorIgnored, "Number ending with underscores is invalid", -1, false); })?
// !!! Error Alternative !!!
- | Zero ([0-9] { invalidDigitCount++; })+ { require(errorIgnored, "Invalid octal number", -(invalidDigitCount + 1), false); } IntegerTypeSuffix?
- ;
-
-fragment
-Zero
- : '0'
+ | '0' [0-9]+ { require(errorIgnored, "Invalid octal number", -getText().length(), false); } IntegerTypeSuffix?
;
fragment
@@ -553,35 +551,18 @@ IntegerTypeSuffix
fragment
DecimalNumeral
- : Zero
- | NonZeroDigit (Digits? | Underscores Digits)
+ : '0'
+ | [1-9] (Digits? | Underscores Digits)
;
fragment
Digits
- : Digit (DigitOrUnderscore* Digit)?
- ;
-
-fragment
-Digit
- : Zero
- | NonZeroDigit
- ;
-
-fragment
-NonZeroDigit
- : [1-9]
- ;
-
-fragment
-DigitOrUnderscore
- : Digit
- | Underscore
+ : [0-9] ([0-9_]* [0-9])?
;
fragment
Underscores
- : Underscore+
+ : '_'+
;
fragment
@@ -591,12 +572,12 @@ Underscore
fragment
HexNumeral
- : Zero [xX] HexDigits
+ : '0' [xX] HexDigits
;
fragment
HexDigits
- : HexDigit (HexDigitOrUnderscore* HexDigit)?
+ : HexDigit ([0-9a-fA-F_]* HexDigit)?
;
fragment
@@ -604,52 +585,24 @@ HexDigit
: [0-9a-fA-F]
;
-fragment
-HexDigitOrUnderscore
- : HexDigit
- | Underscore
- ;
-
fragment
OctalNumeral
- : Zero Underscores? OctalDigits
+ : '0' '_'* OctalDigits
;
fragment
OctalDigits
- : OctalDigit (OctalDigitOrUnderscore* OctalDigit)?
- ;
-
-fragment
-OctalDigit
- : [0-7]
- ;
-
-fragment
-OctalDigitOrUnderscore
- : OctalDigit
- | Underscore
+ : [0-7] ([0-7_]* [0-7])?
;
fragment
BinaryNumeral
- : Zero [bB] BinaryDigits
+ : '0' [bB] BinaryDigits
;
fragment
BinaryDigits
- : BinaryDigit (BinaryDigitOrUnderscore* BinaryDigit)?
- ;
-
-fragment
-BinaryDigit
- : [01]
- ;
-
-fragment
-BinaryDigitOrUnderscore
- : BinaryDigit
- | Underscore
+ : [01] ([01_]* [01])?
;
// §3.10.2 Floating-Point Literals
@@ -662,29 +615,14 @@ FloatingPointLiteral
fragment
DecimalFloatingPointLiteral
- : Digits? Dot Digits ExponentPart? FloatTypeSuffix?
+ : Digits? '.' Digits ExponentPart? FloatTypeSuffix?
| Digits ExponentPart FloatTypeSuffix?
| Digits FloatTypeSuffix
;
fragment
ExponentPart
- : ExponentIndicator SignedInteger
- ;
-
-fragment
-ExponentIndicator
- : [eE]
- ;
-
-fragment
-SignedInteger
- : Sign? Digits
- ;
-
-fragment
-Sign
- : [+\-]
+ : [eE] [+\-]? Digits
;
fragment
@@ -699,22 +637,13 @@ HexadecimalFloatingPointLiteral
fragment
HexSignificand
- : HexNumeral Dot?
- | Zero [xX] HexDigits? Dot HexDigits
+ : HexNumeral '.'?
+ | '0' [xX] HexDigits? '.' HexDigits
;
fragment
BinaryExponent
- : BinaryExponentIndicator SignedInteger
- ;
-
-fragment
-BinaryExponentIndicator
- : [pP]
- ;
-
-fragment
-Dot : '.'
+ : [pP] [+\-]? Digits
;
// §3.10.3 Boolean Literals
@@ -739,9 +668,9 @@ EscapeSequence
fragment
OctalEscape
- : Backslash OctalDigit
- | Backslash OctalDigit OctalDigit
- | Backslash ZeroToThree OctalDigit OctalDigit
+ : Backslash [0-7]
+ | Backslash [0-7] [0-7]
+ | Backslash [0-3] [0-7] [0-7]
;
// Groovy allows 1 or more u's after the backslash
@@ -750,11 +679,6 @@ UnicodeEscape
: Backslash 'u' HexDigit HexDigit HexDigit HexDigit
;
-fragment
-ZeroToThree
- : [0-3]
- ;
-
// Groovy Escape Sequences
fragment
@@ -886,7 +810,7 @@ RBRACK : ']' { this.exitParen(); } -> popMode;
SEMI : ';';
COMMA : ',';
-DOT : Dot;
+DOT : '.';
// §3.12 Operators
@@ -930,12 +854,27 @@ ELVIS_ASSIGN : '?=';
// §3.8 Identifiers (must appear after all keywords in the grammar)
+// ASCII is a pure-DFA split ([A-Z] vs [a-z$_]); Unicode still needs predicates.
CapitalizedIdentifier
- : JavaLetter {Character.isUpperCase(_input.LA(-1))}? JavaLetterOrDigit*
+ : [A-Z] JavaLetterOrDigit*
+ | ~[\u0000-\u007F\uD800-\uDBFF]
+ { isJavaIdentifierStartAndNotIdentifierIgnorable(_input.LA(-1)) && Character.isUpperCase(_input.LA(-1)) }?
+ JavaLetterOrDigit*
+ | [\uD800-\uDBFF] [\uDC00-\uDFFF]
+ { Character.isJavaIdentifierStart(Character.toCodePoint((char) _input.LA(-2), (char) _input.LA(-1)))
+ && Character.isUpperCase(Character.toCodePoint((char) _input.LA(-2), (char) _input.LA(-1))) }?
+ JavaLetterOrDigit*
;
Identifier
- : JavaLetter JavaLetterOrDigit*
+ : [a-z$_] JavaLetterOrDigit*
+ | ~[\u0000-\u007F\uD800-\uDBFF]
+ { isJavaIdentifierStartAndNotIdentifierIgnorable(_input.LA(-1)) && !Character.isUpperCase(_input.LA(-1)) }?
+ JavaLetterOrDigit*
+ | [\uD800-\uDBFF] [\uDC00-\uDFFF]
+ { Character.isJavaIdentifierStart(Character.toCodePoint((char) _input.LA(-2), (char) _input.LA(-1)))
+ && !Character.isUpperCase(Character.toCodePoint((char) _input.LA(-2), (char) _input.LA(-1))) }?
+ JavaLetterOrDigit*
;
fragment
diff --git a/src/antlr/GroovyParser.g4 b/src/antlr/GroovyParser.g4
index 403433c533..f2b6a1f4a9 100644
--- a/src/antlr/GroovyParser.g4
+++ b/src/antlr/GroovyParser.g4
@@ -28,8 +28,8 @@
*/
/**
- * The Groovy grammar is based on the official grammar for Java:
- * https://github.com/antlr/grammars-v4/blob/master/java/Java.g4
+ * The Groovy grammar is based on the optimized Java grammar:
+ * https://github.com/antlr/grammars-v4/tree/master/java/java
*/
parser grammar GroovyParser;
@@ -527,8 +527,10 @@ annotation
;
elementValues
+ // Gated like the optimized Java grammar's IsNotIdentifierAssign: `@Foo(a = 1)`
+ // is named pairs only, so AdaptivePredict does not also explore assignment.
: elementValuePairs
- | elementValue
+ | { !SemanticPredicates.isIdentifierAssign(_input) }? elementValue
;
annotationName : qualifiedClassName ;
@@ -546,7 +548,6 @@ elementValuePairName
| keywords
;
-// TODO verify the potential performance issue because rule expression contains sub-rule assignments(https://github.com/antlr/grammars-v4/issues/215)
elementValue
: elementValueArrayInitializer
| annotation
diff --git a/src/main/java/org/apache/groovy/parser/antlr4/SemanticPredicates.java b/src/main/java/org/apache/groovy/parser/antlr4/SemanticPredicates.java
index ecab4f2c0e..7a0f33e076 100644
--- a/src/main/java/org/apache/groovy/parser/antlr4/SemanticPredicates.java
+++ b/src/main/java/org/apache/groovy/parser/antlr4/SemanticPredicates.java
@@ -25,7 +25,6 @@ import org.codehaus.groovy.GroovyBugError;
import org.codehaus.groovy.ast.ModifierNode;
import java.util.BitSet;
-import java.util.regex.Pattern;
import static org.apache.groovy.parser.antlr4.GroovyParser.ASSIGN;
import static org.apache.groovy.parser.antlr4.GroovyParser.AT;
@@ -44,30 +43,51 @@ import static org.apache.groovy.parser.antlr4.GroovyParser.RPAREN;
import static org.apache.groovy.parser.antlr4.GroovyParser.StringLiteral;
import static org.apache.groovy.parser.antlr4.GroovyParser.WHILE;
import static org.apache.groovy.parser.antlr4.GroovyParser.YIELD;
-import static org.apache.groovy.parser.antlr4.util.StringUtils.matches;
/**
* Some semantic predicates for altering the behaviour of the lexer and parser
*/
public class SemanticPredicates {
- private static final Pattern NONSPACES_PATTERN = Pattern.compile("\\S+?");
- private static final Pattern LETTER_AND_LEFTCURLY_PATTERN = Pattern.compile("[a-zA-Z_{]");
- private static final Pattern NONSURROGATE_PATTERN = Pattern.compile("[^\u0000-\u007F\uD800-\uDBFF]");
- private static final Pattern SURROGATE_PAIR1_PATTERN = Pattern.compile("[\uD800-\uDBFF]");
- private static final Pattern SURROGATE_PAIR2_PATTERN = Pattern.compile("[\uDC00-\uDFFF]");
private static final int PATH_EXPRESSION_ARGUMENTS = 2;
private static final int PATH_EXPRESSION_CLOSURE_OR_LAMBDA = 3;
+ /**
+ * Token types accepted by {@code elementValuePairName} ({@code identifier | keywords}).
+ * Used to distinguish {@code @Foo(a = 1)} (named pairs) from a single element value.
+ */
+ private static final BitSet ELEMENT_VALUE_PAIR_NAME_TYPES = new BitSet();
+ static {
+ int[] types = {
+ Identifier, CapitalizedIdentifier,
+ GroovyParser.ABSTRACT, GroovyParser.AS, GroovyParser.ASSERT, GroovyParser.ASYNC, GroovyParser.AWAIT,
+ GroovyParser.BREAK, GroovyParser.CASE, GroovyParser.CATCH, GroovyParser.CLASS, GroovyParser.CONST,
+ GroovyParser.CONTINUE, GroovyParser.DEF, GroovyParser.DEFAULT, GroovyParser.DEFER, GroovyParser.DO,
+ GroovyParser.ELSE, GroovyParser.ENUM, GroovyParser.EXTENDS, GroovyParser.FINAL, GroovyParser.FINALLY,
+ GroovyParser.FOR, GroovyParser.GOTO, GroovyParser.IF, GroovyParser.IMPLEMENTS, GroovyParser.IMPORT,
+ GroovyParser.IN, GroovyParser.INSTANCEOF, GroovyParser.INTERFACE, GroovyParser.NATIVE, GroovyParser.NEW,
+ GroovyParser.NON_SEALED, GroovyParser.PACKAGE, GroovyParser.PERMITS, GroovyParser.RECORD,
+ GroovyParser.RETURN, GroovyParser.SEALED, GroovyParser.STATIC, GroovyParser.STRICTFP, GroovyParser.SUPER,
+ GroovyParser.SWITCH, GroovyParser.SYNCHRONIZED, GroovyParser.THIS, GroovyParser.THROW, GroovyParser.THROWS,
+ GroovyParser.TRANSIENT, GroovyParser.TRAIT, GroovyParser.THREADSAFE, GroovyParser.TRY, GroovyParser.VAL,
+ GroovyParser.VAR, GroovyParser.VOLATILE, GroovyParser.WHILE, GroovyParser.YIELD,
+ GroovyParser.NullLiteral, GroovyParser.BooleanLiteral, BuiltInPrimitiveType, GroovyParser.VOID,
+ GroovyParser.PUBLIC, GroovyParser.PROTECTED, GroovyParser.PRIVATE, GroovyParser.MODULE
+ };
+ for (int t : types) {
+ ELEMENT_VALUE_PAIR_NAME_TYPES.set(t);
+ }
+ }
+
/**
* Check whether the next characters are only white spaces until the end of line or end of file.
+ * Matches Java regex {@code \s} minus the newlines the loop already stops at: space, tab, VT, FF.
*/
public static boolean isFollowedByWhiteSpaces(CharStream cs) {
- for (int index = 1, c = cs.LA(index); !('\r' == c || '\n' == c || CharStream.EOF == c); index++, c = cs.LA(index)) {
- if (matches(String.valueOf((char) c), NONSPACES_PATTERN)) {
+ for (int index = 1, c = cs.LA(index); c != '\r' && c != '\n' && c != CharStream.EOF; index++, c = cs.LA(index)) {
+ if (c != ' ' && c != '\t' && c != '\f' && c != '\u000B') {
return false;
}
}
-
return true;
}
@@ -92,32 +112,38 @@ public class SemanticPredicates {
public static boolean isFollowedByJavaLetterInGString(CharStream cs) {
int c1 = cs.LA(1);
- if ('$' == c1) { // single $ is not a valid identifier
+ if (c1 == '$' || c1 < 0) { // single $ is not a valid identifier; EOF is not either
return false;
}
- String str1 = String.valueOf((char) c1);
-
- if (matches(str1, LETTER_AND_LEFTCURLY_PATTERN)) {
+ if (c1 == '{' || c1 == '_'
+ || (c1 >= 'A' && c1 <= 'Z')
+ || (c1 >= 'a' && c1 <= 'z')) {
return true;
}
- if (matches(str1, NONSURROGATE_PATTERN)
- && Character.isJavaIdentifierPart(c1)) {
- return true;
+ if (c1 <= 0x7F) {
+ return false;
}
- int c2 = cs.LA(2);
- String str2 = String.valueOf((char) c2);
-
- if (matches(str1, SURROGATE_PAIR1_PATTERN)
- && matches(str2, SURROGATE_PAIR2_PATTERN)
- && Character.isJavaIdentifierPart(Character.toCodePoint((char) c1, (char) c2))) {
-
- return true;
+ if (c1 >= 0xD800 && c1 <= 0xDBFF) {
+ int c2 = cs.LA(2);
+ return c2 >= 0xDC00 && c2 <= 0xDFFF
+ && Character.isJavaIdentifierPart(Character.toCodePoint((char) c1, (char) c2));
}
- return false;
+ return Character.isJavaIdentifierPart(c1);
+ }
+
+ /**
+ * {@code true} when the upcoming tokens are {@code name '=' ...}, i.e. a named
+ * annotation element-value pair rather than a single element value. Inside
+ * annotation parentheses newlines are already hidden, so {@code LT(2)} is the
+ * token after the name.
+ */
+ public static boolean isIdentifierAssign(TokenStream ts) {
+ int t1 = ts.LT(1).getType();
+ return t1 >= 0 && ELEMENT_VALUE_PAIR_NAME_TYPES.get(t1) && ASSIGN == ts.LT(2).getType();
}
/**
diff --git a/src/test/groovy/org/apache/groovy/parser/antlr4/SemanticPredicatesTest.groovy b/src/test/groovy/org/apache/groovy/parser/antlr4/SemanticPredicatesTest.groovy
index d4bb7f11d7..ca81a4d5cf 100644
--- a/src/test/groovy/org/apache/groovy/parser/antlr4/SemanticPredicatesTest.groovy
+++ b/src/test/groovy/org/apache/groovy/parser/antlr4/SemanticPredicatesTest.groovy
@@ -64,6 +64,55 @@ final class SemanticPredicatesTest {
assert !SemanticPredicates.isAnnotatedLoopStatement(tokens('@java.lang.Deprecated String name'))
}
+ @Test
+ void 'identifier assign distinguishes named annotation pairs from single values'() {
+ assert SemanticPredicates.isIdentifierAssign(tokens('a = 1'))
+ assert SemanticPredicates.isIdentifierAssign(tokens('value = 1'))
+ assert SemanticPredicates.isIdentifierAssign(tokens('class = 1'))
+ assert !SemanticPredicates.isIdentifierAssign(tokens('1'))
+ assert !SemanticPredicates.isIdentifierAssign(tokens('a + 1'))
+ assert !SemanticPredicates.isIdentifierAssign(tokens('a'))
+ assert !SemanticPredicates.isIdentifierAssign(tokens('[1]'))
+ assert !SemanticPredicates.isIdentifierAssign(tokens('@Bar'))
+ }
+
+ @Test
+ void 'followed by java letter in GString is a char-class check'() {
+ assert SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString('name'))
+ assert SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString('{x}'))
+ assert SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString('_x'))
+ assert SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString('Ä'))
+ assert !SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString('$x'))
+ assert !SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString('1x'))
+ assert !SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString(''))
+ assert !SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString(' '))
+ }
+
+ @Test
+ void 'followed by whitespaces ignores only ASCII horizontal whitespace'() {
+ assert SemanticPredicates.isFollowedByWhiteSpaces(CharStreams.fromString(''))
+ assert SemanticPredicates.isFollowedByWhiteSpaces(CharStreams.fromString(' \t\f'))
+ assert SemanticPredicates.isFollowedByWhiteSpaces(CharStreams.fromString(' \ncode'))
+ assert !SemanticPredicates.isFollowedByWhiteSpaces(CharStreams.fromString(' x'))
+ assert !SemanticPredicates.isFollowedByWhiteSpaces(CharStreams.fromString('\u00A0'))
+ }
+
+ @Test
+ void 'named and single-element annotations still parse'() {
+ ['@Foo class C {}',
+ '@Foo() class C {}',
+ '@Foo(1) class C {}',
+ '@Foo(a = 1) class C {}',
+ '@Foo(value = 1, other = 2) class C {}',
+ '@Foo(a + 1) class C {}',
+ '@Foo(class = 1) class C {}'
+ ].each { src ->
+ GroovyLangParser p = parser(src)
+ assert p.compilationUnit() != null
+ assert 0 == p.numberOfSyntaxErrors: "Failed to parse `${src}`"
+ }
+ }
+
private static GroovyParser.ExpressionContext parseExpression(String source) {
GroovyLangParser parser = parser(source)
GroovyParser.ExpressionContext expression = parser.expression()
diff --git a/src/test/java/org/apache/groovy/parser/antlr4/AbstractLexerTest.java b/src/test/java/org/apache/groovy/parser/antlr4/AbstractLexerTest.java
index 7ab468df15..bb019ebf64 100644
--- a/src/test/java/org/apache/groovy/parser/antlr4/AbstractLexerTest.java
+++ b/src/test/java/org/apache/groovy/parser/antlr4/AbstractLexerTest.java
@@ -30,6 +30,13 @@ import java.util.Arrays;
import java.util.List;
import java.util.stream.Stream;
+import static org.apache.groovy.parser.antlr4.GroovyLexer.CapitalizedIdentifier;
+import static org.apache.groovy.parser.antlr4.GroovyLexer.FloatingPointLiteral;
+import static org.apache.groovy.parser.antlr4.GroovyLexer.GStringBegin;
+import static org.apache.groovy.parser.antlr4.GroovyLexer.GStringEnd;
+import static org.apache.groovy.parser.antlr4.GroovyLexer.GStringPathPart;
+import static org.apache.groovy.parser.antlr4.GroovyLexer.Identifier;
+import static org.apache.groovy.parser.antlr4.GroovyLexer.IntegerLiteral;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -397,6 +404,49 @@ final class AbstractLexerTest {
assertEquals(Token.EOF, tokens.get(tokens.size() - 1).getType());
}
+ @Test
+ void asciiIdentifierSplitIsPureDfa() {
+ assertEquals(CapitalizedIdentifier, firstDefaultChannel("Foo").getType());
+ assertEquals(CapitalizedIdentifier, firstDefaultChannel("A").getType());
+ assertEquals(Identifier, firstDefaultChannel("foo").getType());
+ assertEquals(Identifier, firstDefaultChannel("$foo").getType());
+ assertEquals(Identifier, firstDefaultChannel("_foo").getType());
+ }
+
+ @Test
+ void unicodeIdentifierSplitFollowsUppercase() {
+ assertEquals(CapitalizedIdentifier, firstDefaultChannel("Äbc").getType());
+ assertEquals(Identifier, firstDefaultChannel("λ").getType());
+ }
+
+ @Test
+ void flattenedNumberLiteralsStillMatch() {
+ assertEquals(IntegerLiteral, firstDefaultChannel("0").getType());
+ assertEquals(IntegerLiteral, firstDefaultChannel("123").getType());
+ assertEquals(IntegerLiteral, firstDefaultChannel("1_000").getType());
+ assertEquals(IntegerLiteral, firstDefaultChannel("0xFF").getType());
+ assertEquals(IntegerLiteral, firstDefaultChannel("0b101").getType());
+ assertEquals(IntegerLiteral, firstDefaultChannel("07").getType());
+ assertEquals(IntegerLiteral, firstDefaultChannel("1G").getType());
+ assertEquals(FloatingPointLiteral, firstDefaultChannel("1.5").getType());
+ assertEquals(FloatingPointLiteral, firstDefaultChannel(".5").getType());
+ assertEquals(FloatingPointLiteral, firstDefaultChannel("1e10").getType());
+ assertEquals(FloatingPointLiteral, firstDefaultChannel("0x1p1").getType());
+ assertEquals(FloatingPointLiteral, firstDefaultChannel("1.5G").getType());
+ }
+
+ @Test
+ void gstringPathStillUsesDotFragmentReplacement() {
+ List<Token> tokens = defaultChannel("\"$foo.bar\"");
+ List<Integer> types = new ArrayList<>();
+ for (Token t : tokens) {
+ types.add(t.getType());
+ }
+ assertTrue(types.contains(GStringBegin), types.toString());
+ assertTrue(types.contains(GStringPathPart), types.toString());
+ assertTrue(types.contains(GStringEnd), types.toString());
+ }
+
private static GroovyLangLexer displayLexer() {
return new GroovyLangLexer(CharStreams.fromString("x"));
}
@@ -419,6 +469,20 @@ final class AbstractLexerTest {
return tokens;
}
+ private static Token firstDefaultChannel(final String src) {
+ return defaultChannel(src).get(0);
+ }
+
+ private static List<Token> defaultChannel(final String src) {
+ List<Token> tokens = new ArrayList<>();
+ for (Token t : collect(src)) {
+ if (t.getType() != Token.EOF && t.getChannel() == Token.DEFAULT_CHANNEL) {
+ tokens.add(t);
+ }
+ }
+ return tokens;
+ }
+
private static List<String> texts(final List<Token> tokens) {
List<String> texts = new ArrayList<>(tokens.size());
for (Token t : tokens) {