You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@royale.apache.org by jo...@apache.org on 2022/12/06 17:13:48 UTC
[royale-compiler] 03/05: compiler: verbatim string tokens are now of type TOKEN_VERBATIM_STRING so that the formatter can know the difference
This is an automated email from the ASF dual-hosted git repository.
joshtynjala pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/royale-compiler.git
commit a71bff565c7853ca9a416118202b0856e8b32f45
Author: Josh Tynjala <jo...@apache.org>
AuthorDate: Tue Dec 6 09:08:07 2022 -0800
compiler: verbatim string tokens are now of type TOKEN_VERBATIM_STRING so that the formatter can know the difference
It still results in the same LiteralNode in the AST. This is purely at the token level.
---
.../apache/royale/compiler/parsing/IASToken.java | 3 ++-
.../royale/compiler/internal/parsing/as/ASParser.g | 4 +++-
.../compiler/internal/parsing/as/ASToken.java | 3 +++
.../compiler/internal/parsing/as/BaseASParser.java | 21 ++++++++-------------
.../compiler/internal/parsing/as/RawASTokenizer.lex | 7 ++++++-
5 files changed, 22 insertions(+), 16 deletions(-)
diff --git a/compiler-common/src/main/java/org/apache/royale/compiler/parsing/IASToken.java b/compiler-common/src/main/java/org/apache/royale/compiler/parsing/IASToken.java
index d2ba7d317..c41d3ed8c 100644
--- a/compiler-common/src/main/java/org/apache/royale/compiler/parsing/IASToken.java
+++ b/compiler-common/src/main/java/org/apache/royale/compiler/parsing/IASToken.java
@@ -60,7 +60,8 @@ public interface IASToken extends ICMToken, ISourceLocation
COLON,
DEFAULT_XML_STATEMENT,
UNKNOWN,
- INCLUDE
+ INCLUDE,
+ VERBATIM_STRING
}
/**
diff --git a/compiler/src/main/antlr/org/apache/royale/compiler/internal/parsing/as/ASParser.g b/compiler/src/main/antlr/org/apache/royale/compiler/internal/parsing/as/ASParser.g
index 90432099a..00a2b8084 100644
--- a/compiler/src/main/antlr/org/apache/royale/compiler/internal/parsing/as/ASParser.g
+++ b/compiler/src/main/antlr/org/apache/royale/compiler/internal/parsing/as/ASParser.g
@@ -2498,8 +2498,10 @@ primaryExpression returns [ExpressionNodeBase n]
{ n = LanguageIdentifierNode.buildThis(token); }
| token=numericLiteral
{ n = new NumericLiteralNode(token); }
+ | TOKEN_VERBATIM_STRING
+ { n = transformRawString(token); }
| TOKEN_LITERAL_STRING
- { n = transformStringLiteral(token); }
+ { n = new LiteralNode(token, LiteralType.STRING); }
| TOKEN_VOID_0
{ n = new LiteralNode(token, LiteralType.OBJECT); }
| TOKEN_LITERAL_REGEXP
diff --git a/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/ASToken.java b/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/ASToken.java
index de34ff3f8..11df97b41 100644
--- a/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/ASToken.java
+++ b/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/ASToken.java
@@ -433,6 +433,7 @@ public class ASToken extends TokenBase implements IASToken, ASTokenTypes
case TOKEN_LITERAL_HEX_NUMBER:
case TOKEN_LITERAL_REGEXP:
case TOKEN_LITERAL_STRING:
+ case TOKEN_VERBATIM_STRING:
case TOKEN_LITERAL_XMLLIST:
case TOKEN_E4X_XMLLIST_CLOSE:
return true;
@@ -937,6 +938,8 @@ public class ASToken extends TokenBase implements IASToken, ASTokenTypes
return ASTokenKind.SCOPE_CLOSE;
case TOKEN_BLOCK_OPEN:
return ASTokenKind.SCOPE_OPEN;
+ case TOKEN_VERBATIM_STRING:
+ return ASTokenKind.VERBATIM_STRING;
case TOKEN_LITERAL_STRING:
return ASTokenKind.STRING_LITERAL;
case TOKEN_LITERAL_NUMBER:
diff --git a/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/BaseASParser.java b/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/BaseASParser.java
index 9a647b3da..7174b62f4 100644
--- a/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/BaseASParser.java
+++ b/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/BaseASParser.java
@@ -3112,21 +3112,16 @@ abstract class BaseASParser extends LLkParser implements IProblemReporter
return parsingProjectConfigVariables;
}
- protected final LiteralNode transformStringLiteral(ASToken token)
+ protected final LiteralNode transformRawString(ASToken token)
{
String tokenText = token.getText();
- if (tokenText.startsWith("@"))
- {
- // the only change needed is to remove the @ symbol
- // the escape sequences are already handled for us
- String newText = tokenText.substring(1);
- ASToken newToken = new ASToken(ASTokenTypes.TOKEN_LITERAL_STRING, token.getStart(), token.getEnd(), token.getLine(), token.getColumn(), newText);
- newToken.setEndLine(token.getEndLine());
- newToken.setEndColumn(token.getEndColumn());
- return new LiteralNode(newToken, LiteralType.STRING);
- }
-
- return new LiteralNode(token, LiteralType.STRING);
+ // the only change needed is to remove the @ symbol
+ // the escape sequences are already handled for us
+ String newText = tokenText.substring(1);
+ ASToken newToken = new ASToken(ASTokenTypes.TOKEN_VERBATIM_STRING, token.getStart(), token.getEnd(), token.getLine(), token.getColumn(), newText);
+ newToken.setEndLine(token.getEndLine());
+ newToken.setEndColumn(token.getEndColumn());
+ return new LiteralNode(newToken, LiteralType.STRING);
}
private final ExpressionNodeBase transformNullishCoalescingExpression(ExpressionNodeBase left, ASToken op, ExpressionNodeBase right)
diff --git a/compiler/src/main/jflex/org/apache/royale/compiler/internal/parsing/as/RawASTokenizer.lex b/compiler/src/main/jflex/org/apache/royale/compiler/internal/parsing/as/RawASTokenizer.lex
index e4a0eee83..ff925c878 100644
--- a/compiler/src/main/jflex/org/apache/royale/compiler/internal/parsing/as/RawASTokenizer.lex
+++ b/compiler/src/main/jflex/org/apache/royale/compiler/internal/parsing/as/RawASTokenizer.lex
@@ -324,11 +324,16 @@ REGEX_CLASS="[" ({REGEX_ESCAPE}|[^\n\r\]\\])* "]"
<STRINGLITERAL> "\""
{
continueAggregate();
- if (stringKind == StringKind.STRING || stringKind == StringKind.RAW)
+ if (stringKind == StringKind.STRING)
{
yybegin(YYINITIAL);
return buildAggregateToken(TOKEN_LITERAL_STRING);
}
+ else if (stringKind == StringKind.RAW)
+ {
+ yybegin(YYINITIAL);
+ return buildAggregateToken(TOKEN_VERBATIM_STRING);
+ }
}
<STRINGLITERAL> "'"