You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@netbeans.apache.org by GitBox <gi...@apache.org> on 2018/12/11 20:59:10 UTC
[GitHub] jlahoda closed pull request #1032: Java lexer changes for raw
string literals.
jlahoda closed pull request #1032: Java lexer changes for raw string literals.
URL: https://github.com/apache/incubator-netbeans/pull/1032
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/java/java.lexer/apichanges.xml b/java/java.lexer/apichanges.xml
index 77a4f2a9cb..26437a1085 100644
--- a/java/java.lexer/apichanges.xml
+++ b/java/java.lexer/apichanges.xml
@@ -83,6 +83,18 @@ is the proper place.
<!-- ACTUAL CHANGES BEGIN HERE: -->
<changes>
+ <change id="RawStringLiteral">
+ <api name="general"/>
+ <summary>Added RAW_STRING_LITERAL JavaTokenKind</summary>
+ <version major="1" minor="40"/>
+ <date day="29" month="11" year="2018"/>
+ <author login="jlahoda"/>
+ <compatibility addition="yes" binary="compatible" deletion="no" deprecation="no" modification="no" semantic="compatible" source="compatible"/>
+ <description>
+ Added JavaTokenId.RAW_STRING_LITERAL.
+ </description>
+ <class package="org.netbeans.api.java.lexer" name="JavaTokenId"/>
+ </change>
<change id="Modules">
<api name="general"/>
<summary>Added tokens used in module-info.java files</summary>
diff --git a/java/java.lexer/manifest.mf b/java/java.lexer/manifest.mf
index e0ffa64324..b569aa224a 100644
--- a/java/java.lexer/manifest.mf
+++ b/java/java.lexer/manifest.mf
@@ -1,5 +1,5 @@
OpenIDE-Module: org.netbeans.modules.java.lexer/1
OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/java/lexer/Bundle.properties
-OpenIDE-Module-Specification-Version: 1.39
+OpenIDE-Module-Specification-Version: 1.40
OpenIDE-Module-Layer: org/netbeans/lib/java/lexer/layer.xml
diff --git a/java/java.lexer/src/org/netbeans/api/java/lexer/JavaTokenId.java b/java/java.lexer/src/org/netbeans/api/java/lexer/JavaTokenId.java
index 7cf0fbb982..d5d561d2ee 100644
--- a/java/java.lexer/src/org/netbeans/api/java/lexer/JavaTokenId.java
+++ b/java/java.lexer/src/org/netbeans/api/java/lexer/JavaTokenId.java
@@ -127,6 +127,8 @@
DOUBLE_LITERAL(null, "number"),
CHAR_LITERAL(null, "character"),
STRING_LITERAL(null, "string"),
+ /**@since 1.40*/
+ RAW_STRING_LITERAL(null, "string"),
TRUE("true", "literal"),
FALSE("false", "literal"),
diff --git a/java/java.lexer/src/org/netbeans/lib/java/lexer/JavaLexer.java b/java/java.lexer/src/org/netbeans/lib/java/lexer/JavaLexer.java
index d0a598df83..1a85893bc0 100644
--- a/java/java.lexer/src/org/netbeans/lib/java/lexer/JavaLexer.java
+++ b/java/java.lexer/src/org/netbeans/lib/java/lexer/JavaLexer.java
@@ -199,6 +199,47 @@ public void consumeNewline() {
input.readLength(), PartType.START);
}
+ case '`': //raw string literal
+ if (input.readLength() > 1) {
+ //even the first backtick must be unencoded for a raw string:
+ return token(JavaTokenId.ERROR);
+ }
+ //detect delimiter:
+ int delimiterCount = 1;
+ while ((c = input.read()) == '`') {
+ delimiterCount++;
+ }
+
+ if (c == EOF) {
+ return tokenFactory.createToken(JavaTokenId.RAW_STRING_LITERAL,
+ input.readLength(), PartType.START);
+ }
+
+ while (true) {
+ while ((c = input.read()) != '`' && c != EOF)
+ ;
+
+ if (c == EOF) {
+ return tokenFactory.createToken(JavaTokenId.RAW_STRING_LITERAL,
+ input.readLength(), PartType.START);
+ }
+
+ int endDelimiterCount = 1;
+
+ while ((c = input.read()) == '`') {
+ endDelimiterCount++;
+ }
+
+ if (delimiterCount == endDelimiterCount) {
+ input.backup(1);
+ return token(JavaTokenId.RAW_STRING_LITERAL);
+ }
+
+ if (c == EOF) {
+ return tokenFactory.createToken(JavaTokenId.RAW_STRING_LITERAL,
+ input.readLength(), PartType.START);
+ }
+ }
case '/':
switch (nextChar()) {
case '/': // in single-line comment
diff --git a/java/java.lexer/test/unit/src/org/netbeans/lib/java/lexer/JavaLexerBatchTest.java b/java/java.lexer/test/unit/src/org/netbeans/lib/java/lexer/JavaLexerBatchTest.java
index b87cbddaf0..a2a239c8cb 100644
--- a/java/java.lexer/test/unit/src/org/netbeans/lib/java/lexer/JavaLexerBatchTest.java
+++ b/java/java.lexer/test/unit/src/org/netbeans/lib/java/lexer/JavaLexerBatchTest.java
@@ -697,4 +697,57 @@ public void testInvalidVarStatement() {
LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
}
+ private void enableRawStringLiterals(InputAttributes attr) {
+ //any setup needed for raw string literals:
+ //e.g.:
+// attr.setValue(JavaTokenId.language(), "version", Integer.valueOf(12), true);
+ //(currently the lexer accepts raw string literals regardless of the source level)
+ }
+
+ public void testRawLiteral() {
+ String text = "`foo bar` `` foo ` bar `` ` `` `";
+ InputAttributes attr = new InputAttributes();
+ enableRawStringLiterals(attr);
+ TokenHierarchy<?> hi = TokenHierarchy.create(text, false, JavaTokenId.language(), EnumSet.noneOf(JavaTokenId.class), attr);
+ TokenSequence<?> ts = hi.tokenSequence();
+
+ LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.RAW_STRING_LITERAL, "`foo bar`");
+ LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
+ LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.RAW_STRING_LITERAL, "`` foo ` bar ``");
+ LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
+ LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.RAW_STRING_LITERAL, "` `` `");
+ }
+
+ public void testUnclosed1() {
+ String text = "```";
+ InputAttributes attr = new InputAttributes();
+ enableRawStringLiterals(attr);
+ TokenHierarchy<?> hi = TokenHierarchy.create(text, false, JavaTokenId.language(), EnumSet.noneOf(JavaTokenId.class), attr);
+ TokenSequence<?> ts = hi.tokenSequence();
+
+ LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.RAW_STRING_LITERAL, "```");
+ }
+
+ public void testEscapes1() {
+ String text = "```\\u0061\\u0060`` ```";
+ InputAttributes attr = new InputAttributes();
+ enableRawStringLiterals(attr);
+ TokenHierarchy<?> hi = TokenHierarchy.create(text, false, JavaTokenId.language(), EnumSet.noneOf(JavaTokenId.class), attr);
+ TokenSequence<?> ts = hi.tokenSequence();
+
+ LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.RAW_STRING_LITERAL, "```\\u0061\\u0060`` ```");
+ }
+
+ public void testEscapesStart() {
+ String text = "\\u0060`\\u0061`` `";
+ InputAttributes attr = new InputAttributes();
+ enableRawStringLiterals(attr);
+ TokenHierarchy<?> hi = TokenHierarchy.create(text, false, JavaTokenId.language(), EnumSet.noneOf(JavaTokenId.class), attr);
+ TokenSequence<?> ts = hi.tokenSequence();
+
+ LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.ERROR, "\\u0060");
+ LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.RAW_STRING_LITERAL, "`\\u0061`` `");
+ }
+
+
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@netbeans.apache.org
For additional commands, e-mail: notifications-help@netbeans.apache.org
For further information about the NetBeans mailing lists, visit:
https://cwiki.apache.org/confluence/display/NETBEANS/Mailing+lists