You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@freemarker.apache.org by dd...@apache.org on 2015/10/02 00:08:49 UTC
[1/2] incubator-freemarker git commit: Simplified static text parsing
Repository: incubator-freemarker
Updated Branches:
refs/heads/2.3-gae 6ac1b86a4 -> d8487ffee
Simplified static text parsing
Project: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/commit/6802cc71
Tree: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/tree/6802cc71
Diff: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/diff/6802cc71
Branch: refs/heads/2.3-gae
Commit: 6802cc7110d796c841f4d49d401dda1c3f78048a
Parents: 6ac1b86
Author: ddekany <dd...@apache.org>
Authored: Thu Oct 1 14:17:11 2015 +0200
Committer: ddekany <dd...@apache.org>
Committed: Thu Oct 1 14:17:11 2015 +0200
----------------------------------------------------------------------
src/main/javacc/FTL.jj | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/6802cc71/src/main/javacc/FTL.jj
----------------------------------------------------------------------
diff --git a/src/main/javacc/FTL.jj b/src/main/javacc/FTL.jj
index f65f4c5..a3c5870 100644
--- a/src/main/javacc/FTL.jj
+++ b/src/main/javacc/FTL.jj
@@ -3893,9 +3893,7 @@ TextBlock PCData() :
}
{
(
- LOOKAHEAD(<STATIC_TEXT_WS>|<STATIC_TEXT_NON_WS>|<STATIC_TEXT_FALSE_ALARM>)
(
- { prevToken = t; }
t = <STATIC_TEXT_WS>
|
t = <STATIC_TEXT_NON_WS>
@@ -3906,6 +3904,7 @@ TextBlock PCData() :
buf.append(t.image);
if (start == null) start = t;
if (prevToken != null) prevToken.next = null;
+ prevToken = t;
}
)+
{
[2/2] incubator-freemarker git commit: Reworked how ${...}-s are
lexed a bit,
to make the lexing of string literals (which may contain ${...}) more
manageable.
Posted by dd...@apache.org.
Reworked how ${...}-s are lexed a bit, to make the lexing of string literals (which may contain ${...}) more manageable.
Project: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/commit/d8487ffe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/tree/d8487ffe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/diff/d8487ffe
Branch: refs/heads/2.3-gae
Commit: d8487ffee38a1fd3ab9291c9174d8f7e7ce15be8
Parents: 6802cc7
Author: ddekany <dd...@apache.org>
Authored: Thu Oct 1 16:20:45 2015 +0200
Committer: ddekany <dd...@apache.org>
Committed: Fri Oct 2 00:07:17 2015 +0200
----------------------------------------------------------------------
src/main/javacc/FTL.jj | 57 +++++++++++++-------
.../core/ParsingErrorMessagesTest.java | 7 +++
.../core/StringLiteralInterpolationTest.java | 55 ++++++++++++-------
3 files changed, 79 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/d8487ffe/src/main/javacc/FTL.jj
----------------------------------------------------------------------
diff --git a/src/main/javacc/FTL.jj b/src/main/javacc/FTL.jj
index a3c5870..9c6bae1 100644
--- a/src/main/javacc/FTL.jj
+++ b/src/main/javacc/FTL.jj
@@ -255,10 +255,11 @@ public class FMParser {
}
void setupStringLiteralMode(FMParserTokenManager parentTokenSource) {
- token_source.onlyTextOutput = true;
token_source.initialNamingConvention = parentTokenSource.initialNamingConvention;
token_source.namingConvention = parentTokenSource.namingConvention;
token_source.namingConventionEstabilisher = parentTokenSource.namingConventionEstabilisher;
+ token_source.SwitchTo(NODIRECTIVE);
+
outputFormat = PlainTextOutputFormat.INSTANCE;
recalculateAutoEscapingField();
if (incompatibleImprovements < _TemplateAPI.VERSION_INT_2_3_24) {
@@ -578,12 +579,12 @@ TOKEN_MGR_DECLS:
* distinguish the } used to close a hash literal and the one used to close a ${
*/
private FMParser parser;
+ private int postInterpolationLexState = -1;
private int hashLiteralNesting;
private int parenthesisNesting;
private int bracketNesting;
private boolean inFTLHeader;
boolean strictEscapeSyntax,
- onlyTextOutput,
squBracTagSyntax,
autodetectTagSyntax,
directiveSyntaxEstablished,
@@ -602,11 +603,6 @@ TOKEN_MGR_DECLS:
// tag syntax detection. If you update this logic, take a look
// at the UNKNOWN_DIRECTIVE token too.
private void strictSyntaxCheck(Token tok, int tokenNamingConvention, int newLexState) {
- if (onlyTextOutput) {
- tok.kind = STATIC_TEXT_NON_WS;
- return;
- }
-
final String image = tok.image;
// Non-strict syntax (deprecated) only supports legacy naming convention.
@@ -731,11 +727,6 @@ TOKEN_MGR_DECLS:
}
private void unifiedCall(Token tok) {
- if (onlyTextOutput) {
- tok.kind = STATIC_TEXT_NON_WS;
- return;
- }
-
char firstChar = tok.image.charAt(0);
if (autodetectTagSyntax && !directiveSyntaxEstablished) {
squBracTagSyntax = (firstChar == '[');
@@ -753,11 +744,6 @@ TOKEN_MGR_DECLS:
}
private void unifiedCallEnd(Token tok) {
- if (onlyTextOutput) {
- tok.kind = STATIC_TEXT_NON_WS;
- return;
- }
-
char firstChar = tok.image.charAt(0);
if (squBracTagSyntax && firstChar == '<') {
tok.kind = STATIC_TEXT_NON_WS;
@@ -781,6 +767,37 @@ TOKEN_MGR_DECLS:
SwitchTo(DEFAULT);
}
}
+
+ private void startInterpolation(Token tok) {
+ if (postInterpolationLexState != -1) {
+ char c = tok.image.charAt(0);
+ throw new TokenMgrError(
+ "You can't start an interpolation (" + c + "{...}) here "
+ + "as you are inside another interpolation.)",
+ TokenMgrError.LEXICAL_ERROR,
+ tok.beginLine, tok.beginColumn,
+ tok.endLine, tok.endColumn);
+ }
+ postInterpolationLexState = curLexState;
+ SwitchTo(FM_EXPRESSION);
+ }
+
+ /**
+ * @param tok
+ * Assumed to be an '}', or something that is the closing pair of another "mirror image" character.
+ */
+ private void endInterpolation(Token tok) {
+ if (postInterpolationLexState == -1) {
+ char c = tok.image.charAt(0);
+ throw new TokenMgrError(
+ "You can't have an \"" + c + "\" here, as there's nothing open that it could close.",
+ TokenMgrError.LEXICAL_ERROR,
+ tok.beginLine, tok.beginColumn,
+ tok.endLine, tok.endColumn);
+ }
+ SwitchTo(postInterpolationLexState);
+ postInterpolationLexState = -1;
+ }
private void eatNewline() {
int charsRead = 0;
@@ -1096,9 +1113,9 @@ TOKEN:
|
<STATIC_TEXT_FALSE_ALARM : "$" | "#" | "<" | "[" | "{"> // to handle a lone dollar sign or "<" or "# or <@ with whitespace after"
|
- <DOLLAR_INTERPOLATION_OPENING : "${"> : FM_EXPRESSION
+ <DOLLAR_INTERPOLATION_OPENING : "${"> { startInterpolation(matchedToken); }
|
- <HASH_INTERPOLATION_OPENING : "#{"> : FM_EXPRESSION
+ <HASH_INTERPOLATION_OPENING : "#{"> { startInterpolation(matchedToken); }
}
<FM_EXPRESSION, IN_PAREN, NAMED_PARAMETER_EXPRESSION> SKIP :
@@ -1259,7 +1276,7 @@ TOKEN:
|
<CLOSING_CURLY_BRACKET : "}">
{
- if (hashLiteralNesting == 0) SwitchTo(DEFAULT);
+ if (hashLiteralNesting == 0) endInterpolation(matchedToken);
else --hashLiteralNesting;
}
|
http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/d8487ffe/src/test/java/freemarker/core/ParsingErrorMessagesTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/freemarker/core/ParsingErrorMessagesTest.java b/src/test/java/freemarker/core/ParsingErrorMessagesTest.java
index d528675..f34ddf2 100644
--- a/src/test/java/freemarker/core/ParsingErrorMessagesTest.java
+++ b/src/test/java/freemarker/core/ParsingErrorMessagesTest.java
@@ -74,6 +74,13 @@ public class ParsingErrorMessagesTest {
assertErrorContains("${blah", "\"{\"", "unclosed");
}
+ @Test
+ public void testInterpolatingClosingsErrors() {
+ assertErrorContains("${x", "unclosed");
+ assertErrorContains("<#assign x = x}>", "\"}\"", "open");
+ // TODO assertErrorContains("<#assign x = '${x'>", "unclosed");
+ }
+
private void assertErrorContains(String ftl, String... expectedSubstrings) {
assertErrorContains(false, ftl, expectedSubstrings);
assertErrorContains(true, ftl, expectedSubstrings);
http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/d8487ffe/src/test/java/freemarker/core/StringLiteralInterpolationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/freemarker/core/StringLiteralInterpolationTest.java b/src/test/java/freemarker/core/StringLiteralInterpolationTest.java
index e6d86e4..fd6fb14 100644
--- a/src/test/java/freemarker/core/StringLiteralInterpolationTest.java
+++ b/src/test/java/freemarker/core/StringLiteralInterpolationTest.java
@@ -26,28 +26,33 @@ import freemarker.template.Configuration;
import freemarker.template.TemplateException;
import freemarker.test.TemplateTest;
+@SuppressWarnings("boxing")
public class StringLiteralInterpolationTest extends TemplateTest {
@Test
public void basics() throws IOException, TemplateException {
- assertOutput("<#assign x = 1>${'${x}'}", "1");
- assertOutput("<#assign x = 1>${'${x} ${x}'}", "1 1");
- assertOutput("<#assign x = 1>${'$\\{x}'}", "${x}");
- assertOutput("<#assign x = 1>${'$\\{x} $\\{x}'}", "${x} ${x}");
- assertOutput("<#assign x = 1>${'<#-- not a comment -->${x}'}", "<#-- not a comment -->1");
- assertOutput("<#assign x = 1>${'<#-- not a comment -->$\\{x}'}", "<#-- not a comment -->${x}");
- assertOutput("<#assign x = 1>${'<@x/>${x}'}", "<@x/>1");
- assertOutput("<#assign x = 1>${'<@x/>$\\{x}'}", "<@x/>${x}");
- assertOutput("<#assign x = 1>${'<@ ${x}<@'}", "<@ 1<@");
- assertOutput("<#assign x = 1>${'<@ $\\{x}<@'}", "<@ ${x}<@");
- assertOutput("<#assign x = 1>${'</...@x>${x}'}", "</...@x>1");
- assertOutput("<#assign x = 1>${'</...@x>$\\{x}'}", "</...@x>${x}");
- assertOutput("<#assign x = 1>${'</@ ${x}</@'}", "</@ 1</@");
- assertOutput("<#assign x = 1>${'</@ $\\{x}</@'}", "</@ ${x}</@");
- assertOutput("<#assign x = 1>${'[@ ${x}'}", "[@ 1");
- assertOutput("<#assign x = 1>${'[@ $\\{x}'}", "[@ ${x}");
- assertOutput("<#assign x = 1>${'<#assign x = 2> ${x}'}", "<#assign x = 2> 1");
- assertOutput("<#assign x = 1>${'<#assign x = 2> $\\{x}'}", "<#assign x = 2> ${x}");
+ addToDataModel("x", 1);
+ assertOutput("${'${x}'}", "1");
+ assertOutput("${'#{x}'}", "1");
+ assertOutput("${'a${x}b${x*2}c'}", "a1b2c");
+ assertOutput("${'a#{x}b#{x*2}c'}", "a1b2c");
+ assertOutput("${'${x} ${x}'}", "1 1");
+ assertOutput("${'$\\{x}'}", "${x}");
+ assertOutput("${'$\\{x} $\\{x}'}", "${x} ${x}");
+ assertOutput("${'<#-- not a comment -->${x}'}", "<#-- not a comment -->1");
+ assertOutput("${'<#-- not a comment -->$\\{x}'}", "<#-- not a comment -->${x}");
+ assertOutput("${'<#assign x = 2> ${x} <#assign x = 2>'}", "<#assign x = 2> 1 <#assign x = 2>");
+ assertOutput("${'<#assign x = 2> $\\{x} <#assign x = 2>'}", "<#assign x = 2> ${x} <#assign x = 2>");
+ assertOutput("${'<@x/>${x}<@x/>'}", "<@x/>1<@x/>");
+ assertOutput("${'<@x/>$\\{x}<@x/>'}", "<@x/>${x}<@x/>");
+ assertOutput("${'<@ ${x}<@'}", "<@ 1<@");
+ assertOutput("${'<@ $\\{x}<@'}", "<@ ${x}<@");
+ assertOutput("${'</...@x>${x}'}", "</...@x>1");
+ assertOutput("${'</...@x>$\\{x}'}", "</...@x>${x}");
+ assertOutput("${'</@ ${x}</@'}", "</@ 1</@");
+ assertOutput("${'</@ $\\{x}</@'}", "</@ ${x}</@");
+ assertOutput("${'[@ ${x}'}", "[@ 1");
+ assertOutput("${'[@ $\\{x}'}", "[@ ${x}");
}
/**
@@ -55,8 +60,18 @@ public class StringLiteralInterpolationTest extends TemplateTest {
*/
@Test
public void legacyEscapingBugStillPresent() throws IOException, TemplateException {
- assertOutput("<#assign x = 1>${'$\\{x} ${x}'}", "1 1");
- assertOutput("<#assign x = 1>${'${x} $\\{x}'}", "1 1");
+ addToDataModel("x", 1);
+ assertOutput("${'$\\{x} ${x}'}", "1 1");
+ assertOutput("${'${x} $\\{x}'}", "1 1");
+ }
+
+ @Test
+ public void legacyLengthGlitch() throws IOException, TemplateException {
+ assertOutput("${'${'}", "${");
+ assertOutput("${'${1'}", "${1");
+ assertOutput("${'${}'}", "${}");
+ assertOutput("${'${1}'}", "1");
+ assertErrorContains("${'${ '}", "");
}
@Test