You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@groovy.apache.org by su...@apache.org on 2026/09/11 15:54:13 UTC
(groovy) branch GROOVY-12398 updated: GROOVY-12398: classify supplementary-plane identifiers by code point
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch GROOVY-12398
in repository https://gitbox.apache.org/repos/asf/groovy.git
The following commit(s) were added to refs/heads/GROOVY-12398 by this push:
new e27a4dabbd GROOVY-12398: classify supplementary-plane identifiers by code point
e27a4dabbd is described below
commit e27a4dabbd3ca09292dbfab8be52b29c69d86ec6
Author: Daniel Sun <su...@apache.org>
AuthorDate: Sat Sep 12 00:53:21 2026 +0900
GROOVY-12398: classify supplementary-plane identifiers by code point
Decode surrogate pairs before isUpperCase so CapitalizedIdentifier is
correct, report invalid octals at the token start, and cover the
annotation pair-name set plus those edge cases with tests.
---
ARCHITECTURE.md | 8 ++-
src/antlr/GroovyLexer.g4 | 39 ++++++++--
src/antlr/GroovyParser.g4 | 3 +
.../apache/groovy/parser/antlr4/AbstractLexer.java | 12 ++++
.../groovy/parser/antlr4/SemanticPredicates.java | 12 +++-
.../parser/antlr4/CommonSyntaxErrorTest.groovy | 22 ++++++
.../groovy/parser/antlr4/Groovy12398Test.groovy | 83 ++++++++++++++++++++++
.../parser/antlr4/SemanticPredicatesTest.groovy | 55 +++++++++++++-
.../groovy/parser/antlr4/AbstractLexerTest.java | 54 +++++++++++++-
9 files changed, 278 insertions(+), 10 deletions(-)
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index 51702c8f29..896f1d7f3b 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -95,7 +95,13 @@ verbatim keeps the reference precise; paraphrasing tends to drift.
- Grammar lives in `src/antlr/GroovyLexer.g4` and
`src/antlr/GroovyParser.g4`. The generated parser is regenerated
from these sources on every build, so changes belong in the `.g4`
- files.
+ files. Lexer number literals use character classes rather than
+ one-character fragment chains (optimized Java grammar style). ASCII
+ identifiers are a predicate-free DFA split (`A-Z` vs `a-z$_`);
+ supplementary-plane letters classify capitalized vs not from the
+ decoded code point, not `LA(-1)` (a low surrogate is never
+ uppercase). Annotation `elementValues` is gated so `AdaptivePredict`
+ does not also explore assignment (GROOVY-12398).
- The ANTLR Gradle plugin generates `GroovyLexer`, `GroovyParser`,
`GroovyParserVisitor`, and `GroovyParserBaseVisitor` into
`build/generated/sources/antlr4/org/apache/groovy/parser/antlr4/`.
diff --git a/src/antlr/GroovyLexer.g4 b/src/antlr/GroovyLexer.g4
index 252195574c..12f228a7d0 100644
--- a/src/antlr/GroovyLexer.g4
+++ b/src/antlr/GroovyLexer.g4
@@ -253,6 +253,31 @@ options {
return Character.isJavaIdentifierPart(codePoint) && !Character.isIdentifierIgnorable(codePoint);
}
+ /**
+ * Code point just consumed. After a UTF-16 surrogate pair, {@code LA(-1)}
+ * is the low surrogate and is never uppercase — {@code Character.isUpperCase}
+ * must see the pair (GROOVY-12398).
+ */
+ private int lastConsumedCodePoint() {
+ int c1 = _input.LA(-1);
+ int c2 = _input.LA(-2);
+ if (c1 >= Character.MIN_LOW_SURROGATE && c1 <= Character.MAX_LOW_SURROGATE
+ && c2 >= Character.MIN_HIGH_SURROGATE && c2 <= Character.MAX_HIGH_SURROGATE) {
+ return Character.toCodePoint((char) c2, (char) c1);
+ }
+ return c1;
+ }
+
+ private boolean isSupplementaryCapitalizedIdentifierStart() {
+ int cp = lastConsumedCodePoint();
+ return Character.isJavaIdentifierStart(cp) && Character.isUpperCase(cp);
+ }
+
+ private boolean isSupplementaryUncapitalizedIdentifierStart() {
+ int cp = lastConsumedCodePoint();
+ return Character.isJavaIdentifierStart(cp) && !Character.isUpperCase(cp);
+ }
+
public boolean isErrorIgnored() {
return errorIgnored;
}
@@ -521,7 +546,7 @@ IntegerLiteral
) (Underscore { require(errorIgnored, "Number ending with underscores is invalid", -1, false); })?
// !!! Error Alternative !!!
- | '0' [0-9]+ { require(errorIgnored, "Invalid octal number", -getText().length(), false); } IntegerTypeSuffix?
+ | '0' [0-9]+ { requireInvalidOctal(errorIgnored); } IntegerTypeSuffix?
;
fragment
@@ -854,15 +879,18 @@ ELVIS_ASSIGN : '?=';
// §3.8 Identifiers (must appear after all keywords in the grammar)
-// ASCII is a pure-DFA split ([A-Z] vs [a-z$_]); Unicode still needs predicates.
+// ASCII is a pure-DFA split ([A-Z] vs [a-z$_]) — no predicate on that hot path.
+// BMP non-ASCII still uses LA(-1). Supplementary-plane letters must use the
+// decoded code point: LA(-1) after a surrogate pair is the low surrogate,
+// which is never uppercase, so the old isUpperCase(LA(-1)) test silently
+// classified every supplementary identifier as Identifier (GROOVY-12398).
CapitalizedIdentifier
: [A-Z] JavaLetterOrDigit*
| ~[\u0000-\u007F\uD800-\uDBFF]
{ isJavaIdentifierStartAndNotIdentifierIgnorable(_input.LA(-1)) && Character.isUpperCase(_input.LA(-1)) }?
JavaLetterOrDigit*
| [\uD800-\uDBFF] [\uDC00-\uDFFF]
- { Character.isJavaIdentifierStart(Character.toCodePoint((char) _input.LA(-2), (char) _input.LA(-1)))
- && Character.isUpperCase(Character.toCodePoint((char) _input.LA(-2), (char) _input.LA(-1))) }?
+ { isSupplementaryCapitalizedIdentifierStart() }?
JavaLetterOrDigit*
;
@@ -872,8 +900,7 @@ Identifier
{ isJavaIdentifierStartAndNotIdentifierIgnorable(_input.LA(-1)) && !Character.isUpperCase(_input.LA(-1)) }?
JavaLetterOrDigit*
| [\uD800-\uDBFF] [\uDC00-\uDFFF]
- { Character.isJavaIdentifierStart(Character.toCodePoint((char) _input.LA(-2), (char) _input.LA(-1)))
- && !Character.isUpperCase(Character.toCodePoint((char) _input.LA(-2), (char) _input.LA(-1))) }?
+ { isSupplementaryUncapitalizedIdentifierStart() }?
JavaLetterOrDigit*
;
diff --git a/src/antlr/GroovyParser.g4 b/src/antlr/GroovyParser.g4
index f2b6a1f4a9..3a53de79c1 100644
--- a/src/antlr/GroovyParser.g4
+++ b/src/antlr/GroovyParser.g4
@@ -1275,6 +1275,8 @@ className
: CapitalizedIdentifier
;
+// Tokens added here or to keywords must also be listed in
+// SemanticPredicates.ELEMENT_VALUE_PAIR_NAME_TYPES (annotation name = value).
identifier
: Identifier
| CapitalizedIdentifier
@@ -1299,6 +1301,7 @@ builtInType
;
keywords
+ // Keep ELEMENT_VALUE_PAIR_NAME_TYPES in sync when adding alternatives.
: ABSTRACT
| AS
| ASSERT
diff --git a/src/main/java/org/apache/groovy/parser/antlr4/AbstractLexer.java b/src/main/java/org/apache/groovy/parser/antlr4/AbstractLexer.java
index f755682cf2..4057eff850 100644
--- a/src/main/java/org/apache/groovy/parser/antlr4/AbstractLexer.java
+++ b/src/main/java/org/apache/groovy/parser/antlr4/AbstractLexer.java
@@ -213,6 +213,18 @@ public abstract class AbstractLexer extends Lexer implements SyntaxErrorReportab
requireAtTokenStart(errorIgnored, "Unclosed comment");
}
+ /**
+ * Invalid octal ({@code 08}, {@code 09}, {@code 08L}) reports at the
+ * token start. Using the start index rather than a running digit count keeps
+ * the caret correct for a later invalid octal in the same file, and for a
+ * suffix after the digits.
+ *
+ * @param errorIgnored when {@code true}, keep tokenising (IDE highlighting)
+ */
+ void requireInvalidOctal(final boolean errorIgnored) {
+ requireAtTokenStart(errorIgnored, "Invalid octal number");
+ }
+
/**
* {@link SyntaxErrorReportable#require} positions relative to the current
* lexer cursor. After scanning to EOF that cursor is past the comment, so
diff --git a/src/main/java/org/apache/groovy/parser/antlr4/SemanticPredicates.java b/src/main/java/org/apache/groovy/parser/antlr4/SemanticPredicates.java
index 7a0f33e076..5127eb4271 100644
--- a/src/main/java/org/apache/groovy/parser/antlr4/SemanticPredicates.java
+++ b/src/main/java/org/apache/groovy/parser/antlr4/SemanticPredicates.java
@@ -45,7 +45,15 @@ import static org.apache.groovy.parser.antlr4.GroovyParser.WHILE;
import static org.apache.groovy.parser.antlr4.GroovyParser.YIELD;
/**
- * Some semantic predicates for altering the behaviour of the lexer and parser
+ * Semantic predicates for the lexer and parser.
+ * <p>
+ * Lexer helpers on this class run on the tokenisation hot path (GString
+ * {@code $}, slashy strings, newline-as-separator). They use direct character
+ * tests, not {@link java.util.regex.Pattern}. Parser helpers such as
+ * {@link #isIdentifierAssign(TokenStream)} are O(1) bitset lookups so
+ * {@code AdaptivePredict} can skip exploring assignment as an annotation
+ * element value (GROOVY-12398).
+ * </p>
*/
public class SemanticPredicates {
private static final int PATH_EXPRESSION_ARGUMENTS = 2;
@@ -54,6 +62,8 @@ public class SemanticPredicates {
/**
* Token types accepted by {@code elementValuePairName} ({@code identifier | keywords}).
* Used to distinguish {@code @Foo(a = 1)} (named pairs) from a single element value.
+ * Must stay in sync with those two parser rules; {@code MODULE} is in
+ * {@code identifier} but not in {@code keywords}.
*/
private static final BitSet ELEMENT_VALUE_PAIR_NAME_TYPES = new BitSet();
static {
diff --git a/src/test/groovy/org/apache/groovy/parser/antlr4/CommonSyntaxErrorTest.groovy b/src/test/groovy/org/apache/groovy/parser/antlr4/CommonSyntaxErrorTest.groovy
index 45e04782d3..36ea368c96 100644
--- a/src/test/groovy/org/apache/groovy/parser/antlr4/CommonSyntaxErrorTest.groovy
+++ b/src/test/groovy/org/apache/groovy/parser/antlr4/CommonSyntaxErrorTest.groovy
@@ -316,6 +316,28 @@ final class CommonSyntaxErrorTest {
expectContainsOnce 'def n = 08', 'Invalid octal number'
}
+ @Test
+ void 'invalid octal number caret is the digit not a drifted count'() {
+ expectParseError 'def n = 08', '''\
+ |Invalid octal number @ line 1, column 9.
+ | def n = 08
+ | ^
+ |
+ |1 error
+ |'''.stripMargin()
+ }
+
+ @Test
+ void 'invalid octal with suffix caret is still the token start'() {
+ expectParseError 'def n = 08L', '''\
+ |Invalid octal number @ line 1, column 9.
+ | def n = 08L
+ | ^
+ |
+ |1 error
+ |'''.stripMargin()
+ }
+
@Test
void 'shebang not on the first line'() {
expectContainsOnce 'x = 1\n#!/usr/bin/env groovy', 'Shebang comment should appear at the first line'
diff --git a/src/test/groovy/org/apache/groovy/parser/antlr4/Groovy12398Test.groovy b/src/test/groovy/org/apache/groovy/parser/antlr4/Groovy12398Test.groovy
new file mode 100644
index 0000000000..7b8439f0e8
--- /dev/null
+++ b/src/test/groovy/org/apache/groovy/parser/antlr4/Groovy12398Test.groovy
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.groovy.parser.antlr4
+
+import org.codehaus.groovy.ast.ClassNode
+import org.codehaus.groovy.ast.ModuleNode
+import org.codehaus.groovy.ast.expr.DeclarationExpression
+import org.codehaus.groovy.ast.expr.MethodCallExpression
+import org.codehaus.groovy.ast.stmt.ExpressionStatement
+import org.codehaus.groovy.control.CompilationFailedException
+import org.codehaus.groovy.control.CompilationUnit
+import org.codehaus.groovy.control.Phases
+import org.junit.jupiter.api.Test
+
+import static groovy.test.GroovyAssert.shouldFail
+import static org.junit.jupiter.api.Assertions.assertEquals
+import static org.junit.jupiter.api.Assertions.assertInstanceOf
+
+/**
+ * GROOVY-12398: supplementary-plane letters classify as
+ * {@code CapitalizedIdentifier} from the decoded code point, so
+ * {@code className} (which accepts only that token) can use them.
+ */
+final class Groovy12398Test {
+
+ // DESERET CAPITAL LETTER LONG I / DESERET SMALL LETTER LONG I
+ private static final String UPPER = "\uD801\uDC00"
+ private static final String LOWER = "\uD801\uDC28"
+
+ @Test
+ void 'supplementary uppercase is a type-parameter class name'() {
+ ModuleNode ast = module("class C<${UPPER}> {}")
+ ClassNode c = ast.classes.find { it.nameWithoutPackage == 'C' }
+ assertEquals 1, c.genericsTypes.length
+ assertEquals UPPER, c.genericsTypes[0].name
+ }
+
+ @Test
+ void 'supplementary lowercase is not a type-parameter class name'() {
+ shouldFail(CompilationFailedException) {
+ module("class C<${LOWER}> {}")
+ }
+ }
+
+ @Test
+ void 'supplementary uppercase local variable is a declaration'() {
+ ModuleNode ast = module("${UPPER} x = 1")
+ def stmt = ast.statementBlock.statements[0]
+ assertInstanceOf(ExpressionStatement, stmt)
+ assertInstanceOf(DeclarationExpression, stmt.expression)
+ }
+
+ @Test
+ void 'supplementary lowercase leading identifier is a command expression'() {
+ ModuleNode ast = module("${LOWER} x")
+ def stmt = ast.statementBlock.statements[0]
+ assertInstanceOf(ExpressionStatement, stmt)
+ assertInstanceOf(MethodCallExpression, stmt.expression)
+ }
+
+ private static ModuleNode module(String src) {
+ def cu = new CompilationUnit()
+ cu.addSource('test.groovy', src)
+ cu.compile(Phases.CONVERSION)
+ return cu.AST.modules[0]
+ }
+}
diff --git a/src/test/groovy/org/apache/groovy/parser/antlr4/SemanticPredicatesTest.groovy b/src/test/groovy/org/apache/groovy/parser/antlr4/SemanticPredicatesTest.groovy
index ca81a4d5cf..cfe48aa9b7 100644
--- a/src/test/groovy/org/apache/groovy/parser/antlr4/SemanticPredicatesTest.groovy
+++ b/src/test/groovy/org/apache/groovy/parser/antlr4/SemanticPredicatesTest.groovy
@@ -74,6 +74,28 @@ final class SemanticPredicatesTest {
assert !SemanticPredicates.isIdentifierAssign(tokens('a'))
assert !SemanticPredicates.isIdentifierAssign(tokens('[1]'))
assert !SemanticPredicates.isIdentifierAssign(tokens('@Bar'))
+ assert !SemanticPredicates.isIdentifierAssign(tokens(''))
+ assert !SemanticPredicates.isIdentifierAssign(tokens('(a = 1)'))
+ }
+
+ @Test
+ void 'identifier assign accepts every elementValuePairName token'() {
+ // identifier extras + keywords, including MODULE which is not in keywords.
+ [
+ 'a', 'Foo', 'as', 'async', 'await', 'defer', 'in', 'module', 'permits',
+ 'record', 'sealed', 'trait', 'val', 'var', 'yield',
+ 'abstract', 'assert', 'break', 'case', 'catch', 'class', 'const',
+ 'continue', 'def', 'default', 'do', 'else', 'enum', 'extends', 'final',
+ 'finally', 'for', 'goto', 'if', 'implements', 'import', 'instanceof',
+ 'interface', 'native', 'new', 'non-sealed', 'package', 'return',
+ 'static', 'strictfp', 'super', 'switch', 'synchronized', 'this',
+ 'throw', 'throws', 'transient', 'threadsafe', 'try', 'volatile',
+ 'while', 'null', 'true', 'false', 'int', 'void', 'public', 'protected',
+ 'private'
+ ].each { name ->
+ assert SemanticPredicates.isIdentifierAssign(tokens("${name} = 1")):
+ "expected ${name} = 1 to be a named annotation pair"
+ }
}
@Test
@@ -82,21 +104,46 @@ final class SemanticPredicatesTest {
assert SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString('{x}'))
assert SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString('_x'))
assert SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString('Ä'))
+ assert SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString('\uD801\uDC28')) // Deseret small
assert !SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString('$x'))
assert !SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString('1x'))
assert !SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString(''))
assert !SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString(' '))
+ assert !SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString('\uDC00')) // lone low surrogate
+ assert !SemanticPredicates.isFollowedByJavaLetterInGString(CharStreams.fromString('\uD801')) // high surrogate at EOF
}
@Test
void 'followed by whitespaces ignores only ASCII horizontal whitespace'() {
assert SemanticPredicates.isFollowedByWhiteSpaces(CharStreams.fromString(''))
assert SemanticPredicates.isFollowedByWhiteSpaces(CharStreams.fromString(' \t\f'))
+ assert SemanticPredicates.isFollowedByWhiteSpaces(CharStreams.fromString('\u000B'))
assert SemanticPredicates.isFollowedByWhiteSpaces(CharStreams.fromString(' \ncode'))
assert !SemanticPredicates.isFollowedByWhiteSpaces(CharStreams.fromString(' x'))
assert !SemanticPredicates.isFollowedByWhiteSpaces(CharStreams.fromString('\u00A0'))
}
+ @Test
+ void 'followed by matches any listed character'() {
+ def cs = CharStreams.fromString('[x]')
+ assert SemanticPredicates.isFollowedBy(cs, ' ' as char, '\t' as char, '[' as char)
+ assert !SemanticPredicates.isFollowedBy(cs, ' ' as char, '\t' as char, '(' as char)
+ assert !SemanticPredicates.isFollowedBy(CharStreams.fromString(''), 'a' as char)
+ }
+
+ @Test
+ void 'following arguments or closure is false for non-postfix expressions'() {
+ assert !SemanticPredicates.isFollowingArgumentsOrClosure(parseExpression('1 + 2'))
+ }
+
+ @Test
+ void 'invalid local variable declaration uses the identifier code point'() {
+ // supplementary uppercase: typed declaration, not a command
+ assert !SemanticPredicates.isInvalidLocalVariableDeclaration(tokens('\uD801\uDC00 x = 1'))
+ // supplementary lowercase: command expression
+ assert SemanticPredicates.isInvalidLocalVariableDeclaration(tokens('\uD801\uDC28 x'))
+ }
+
@Test
void 'named and single-element annotations still parse'() {
['@Foo class C {}',
@@ -105,7 +152,13 @@ final class SemanticPredicatesTest {
'@Foo(a = 1) class C {}',
'@Foo(value = 1, other = 2) class C {}',
'@Foo(a + 1) class C {}',
- '@Foo(class = 1) class C {}'
+ '@Foo(class = 1) class C {}',
+ '@Foo(module = 1) class C {}',
+ '@Foo(non-sealed = 1) class C {}',
+ '@Foo(int = 1) class C {}',
+ '@Foo([1, 2]) class C {}',
+ '@Foo(@Bar) class C {}',
+ '@Foo(null) class C {}'
].each { src ->
GroovyLangParser p = parser(src)
assert p.compilationUnit() != null
diff --git a/src/test/java/org/apache/groovy/parser/antlr4/AbstractLexerTest.java b/src/test/java/org/apache/groovy/parser/antlr4/AbstractLexerTest.java
index f6377e1f7d..0a1d2e9ef8 100644
--- a/src/test/java/org/apache/groovy/parser/antlr4/AbstractLexerTest.java
+++ b/src/test/java/org/apache/groovy/parser/antlr4/AbstractLexerTest.java
@@ -417,6 +417,54 @@ final class AbstractLexerTest {
void unicodeIdentifierSplitFollowsUppercase() {
assertEquals(CapitalizedIdentifier, firstDefaultChannel("Äbc").getType());
assertEquals(Identifier, firstDefaultChannel("λ").getType());
+ // title case is not uppercase — still Identifier
+ assertEquals(Identifier, firstDefaultChannel("\u01C5").getType());
+ }
+
+ /**
+ * GROOVY-12398: after a surrogate pair, {@code LA(-1)} is the low
+ * surrogate (never uppercase). Classification must use the code point.
+ * U+10400 DESERET CAPITAL LETTER LONG I / U+10428 DESERET SMALL LETTER LONG I.
+ */
+ @Test
+ void supplementaryPlaneIdentifierSplitUsesCodePoint() {
+ assertEquals(CapitalizedIdentifier, firstDefaultChannel("\uD801\uDC00").getType());
+ assertEquals(Identifier, firstDefaultChannel("\uD801\uDC28").getType());
+ assertEquals(CapitalizedIdentifier, firstDefaultChannel("\uD801\uDC00Name").getType());
+ assertEquals(Identifier, firstDefaultChannel("\uD801\uDC28name").getType());
+ }
+
+ @Test
+ void invalidOctalReportsAtTokenStart() {
+ GroovySyntaxError first = assertThrows(GroovySyntaxError.class,
+ () -> drain(new GroovyLangLexer(CharStreams.fromString("08"))));
+ assertEquals(1, first.getLine());
+ assertEquals(1, first.getColumn());
+
+ GroovySyntaxError withSuffix = assertThrows(GroovySyntaxError.class,
+ () -> drain(new GroovyLangLexer(CharStreams.fromString("08L"))));
+ assertEquals(1, withSuffix.getColumn());
+
+ GroovySyntaxError afterPrefix = assertThrows(GroovySyntaxError.class,
+ () -> drain(new GroovyLangLexer(CharStreams.fromString("x=08"))));
+ assertEquals(3, afterPrefix.getColumn());
+
+ // first of two invalid octals still points at column 1, not a drifted count
+ GroovySyntaxError two = assertThrows(GroovySyntaxError.class,
+ () -> drain(new GroovyLangLexer(CharStreams.fromString("08 09"))));
+ assertEquals(1, two.getColumn());
+ }
+
+ @Test
+ void errorIgnoredTokenizesSuccessiveInvalidOctals() {
+ GroovyLangLexer lexer = new GroovyLangLexer(CharStreams.fromString("08 09 08L"));
+ lexer.setErrorIgnored(true);
+ List<Token> tokens = defaultChannel(lexer);
+ assertEquals(3, tokens.size(), texts(tokens).toString());
+ assertEquals(IntegerLiteral, tokens.get(0).getType());
+ assertEquals(IntegerLiteral, tokens.get(1).getType());
+ assertEquals(IntegerLiteral, tokens.get(2).getType());
+ assertEquals("08L", tokens.get(2).getText());
}
@Test
@@ -474,8 +522,12 @@ final class AbstractLexerTest {
}
private static List<Token> defaultChannel(final String src) {
+ return defaultChannel(new GroovyLangLexer(CharStreams.fromString(src)));
+ }
+
+ private static List<Token> defaultChannel(final GroovyLangLexer lexer) {
List<Token> tokens = new ArrayList<>();
- for (Token t : collect(src)) {
+ for (Token t : collect(lexer)) {
if (t.getType() != Token.EOF && t.getChannel() == Token.DEFAULT_CHANNEL) {
tokens.add(t);
}