You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@freemarker.apache.org by dd...@apache.org on 2020/10/13 19:57:16 UTC
[freemarker] 02/02: Added ?eval_json to evaluate JSON given as flat
string. This was added as ?eval is routinely misused for the same purpose.
This is an automated email from the ASF dual-hosted git repository.
ddekany pushed a commit to branch 2.3-gae
in repository https://gitbox.apache.org/repos/asf/freemarker.git
commit 11c2b09574899f1779ad4941adc1994455d53828
Author: ddekany <dd...@apache.org>
AuthorDate: Tue Oct 13 17:13:39 2020 +0200
Added ?eval_json to evaluate JSON given as flat string. This was added as ?eval is routinely misused for the same purpose.
---
src/main/java/freemarker/core/BuiltIn.java | 4 +-
.../freemarker/core/BuiltInsForStringsMisc.java | 17 +-
src/main/java/freemarker/core/JSONParser.java | 622 +++++++++++++++++++++
src/manual/en_US/book.xml | 98 ++++
.../java/freemarker/core/EvalJsonBuiltInTest.java | 41 ++
src/test/java/freemarker/core/JSONParserTest.java | 171 ++++++
6 files changed, 951 insertions(+), 2 deletions(-)
diff --git a/src/main/java/freemarker/core/BuiltIn.java b/src/main/java/freemarker/core/BuiltIn.java
index 30be937..fcea193 100644
--- a/src/main/java/freemarker/core/BuiltIn.java
+++ b/src/main/java/freemarker/core/BuiltIn.java
@@ -65,6 +65,7 @@ import freemarker.core.BuiltInsForSequences.sequenceBI;
import freemarker.core.BuiltInsForSequences.sortBI;
import freemarker.core.BuiltInsForSequences.sort_byBI;
import freemarker.core.BuiltInsForStringsMisc.evalBI;
+import freemarker.core.BuiltInsForStringsMisc.evalJsonBI;
import freemarker.template.Configuration;
import freemarker.template.TemplateDateModel;
import freemarker.template.TemplateModel;
@@ -84,7 +85,7 @@ abstract class BuiltIn extends Expression implements Cloneable {
static final Set<String> CAMEL_CASE_NAMES = new TreeSet<>();
static final Set<String> SNAKE_CASE_NAMES = new TreeSet<>();
- static final int NUMBER_OF_BIS = 289;
+ static final int NUMBER_OF_BIS = 291;
static final HashMap<String, BuiltIn> BUILT_INS_BY_NAME = new HashMap(NUMBER_OF_BIS * 3 / 2 + 1, 1f);
static final String BI_NAME_SNAKE_CASE_WITH_ARGS = "with_args";
@@ -120,6 +121,7 @@ abstract class BuiltIn extends Expression implements Cloneable {
putBI("ensure_starts_with", "ensureStartsWith", new BuiltInsForStringsBasic.ensure_starts_withBI());
putBI("esc", new escBI());
putBI("eval", new evalBI());
+ putBI("eval_json", "evalJson", new evalJsonBI());
putBI("exists", new BuiltInsForExistenceHandling.existsBI());
putBI("filter", new BuiltInsForSequences.filterBI());
putBI("first", new firstBI());
diff --git a/src/main/java/freemarker/core/BuiltInsForStringsMisc.java b/src/main/java/freemarker/core/BuiltInsForStringsMisc.java
index 012a007..284abee 100644
--- a/src/main/java/freemarker/core/BuiltInsForStringsMisc.java
+++ b/src/main/java/freemarker/core/BuiltInsForStringsMisc.java
@@ -113,6 +113,22 @@ class BuiltInsForStringsMisc {
}
+ static class evalJsonBI extends BuiltInForString {
+ @Override
+ TemplateModel calculateResult(String s, Environment env) throws TemplateException {
+ try {
+ return JSONParser.parse(s);
+ } catch (JSONParser.JSONParseException e) {
+ throw new _MiscTemplateException(this, env,
+ "Failed to \"?", key, "\" string with this error:\n\n",
+ _MessageUtil.EMBEDDED_MESSAGE_BEGIN,
+ new _DelayedGetMessage(e),
+ _MessageUtil.EMBEDDED_MESSAGE_END,
+ "\n\nThe failing expression:");
+ }
+ }
+ }
+
static class numberBI extends BuiltInForString {
@Override
TemplateModel calculateResult(String s, Environment env) throws TemplateException {
@@ -170,5 +186,4 @@ class BuiltInsForStringsMisc {
// Can't be instantiated
private BuiltInsForStringsMisc() { }
-
}
diff --git a/src/main/java/freemarker/core/JSONParser.java b/src/main/java/freemarker/core/JSONParser.java
new file mode 100644
index 0000000..ddb01c0
--- /dev/null
+++ b/src/main/java/freemarker/core/JSONParser.java
@@ -0,0 +1,622 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package freemarker.core;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import freemarker.template.SimpleHash;
+import freemarker.template.SimpleNumber;
+import freemarker.template.SimpleScalar;
+import freemarker.template.SimpleSequence;
+import freemarker.template.Template;
+import freemarker.template.TemplateBooleanModel;
+import freemarker.template.TemplateHashModelEx2;
+import freemarker.template.TemplateModel;
+import freemarker.template.TemplateModelException;
+import freemarker.template.TemplateNumberModel;
+import freemarker.template.TemplateScalarModel;
+import freemarker.template.TemplateSequenceModel;
+import freemarker.template._TemplateAPI;
+import freemarker.template.utility.Constants;
+import freemarker.template.utility.NumberUtil;
+import freemarker.template.utility.StringUtil;
+
+/**
+ * JSON parser that returns a {@link TameplatModel}, similar to what FTL literals product (and so, what
+ * @code ?eval} would return). A notable difference compared to the result FTL literals is that this doesn't use the
+ * {@link ParserConfiguration#getArithmeticEngine()} to parse numbers, as JSON has its own fixed number syntax. For
+ * numbers this parser returns {@link SimpleNumberModel}-s, where the wrapped numbers will be {@link Integer}-s when
+ * they fit into that, otherwise they will be {@link Long}-s if they fit into that, otherwise they will be
+ * {@link BigDecimal}-s. Another difference to the result of FTL literals is that instead of
+ * {@code HashLiteral.SequenceHash} it uses {@link SimpleHash} with {@link LinkedHashMap} as backing store, for
+ * efficiency.
+ *
+ * <p>This parser allows certain things that are errors in pure JSON:
+ * <ul>
+ * <li>JavaScript comments are supported</li>
+ * <li>Non-breaking space (nbsp) and BOM are treated as whitespace</li>
+ * </ul>
+ */
+class JSONParser {
+
+ private static final String UNCLOSED_OBJECT_MESSAGE
+ = "This {...} was still unclosed when the end of the file was reached. (Look for a missing \"}\")";
+
+ private static final String UNCLOSED_ARRAY_MESSAGE
+ = "This [...] was still unclosed when the end of the file was reached. (Look for a missing \"]\")";
+
+ private static final BigDecimal MIN_INT_AS_BIGDECIMAL = BigDecimal.valueOf(Integer.MIN_VALUE);
+ private static final BigDecimal MAX_INT_AS_BIGDECIMAL = BigDecimal.valueOf(Integer.MAX_VALUE);
+ private static final BigDecimal MIN_LONG_AS_BIGDECIMAL = BigDecimal.valueOf(Long.MIN_VALUE);
+ private static final BigDecimal MAX_LONG_AS_BIGDECIMAL = BigDecimal.valueOf(Long.MAX_VALUE);
+
+ private final String src;
+ private final int ln;
+
+ private int p;
+
+ public static TemplateModel parse(String src) throws JSONParseException {
+ return new JSONParser(src).parse();
+ }
+
+ /**
+ * @param sourceLocation Only used in error messages, maybe {@code null}.
+ */
+ private JSONParser(String src) {
+ this.src = src;
+ this.ln = src.length();
+ }
+
+ private TemplateModel parse() throws JSONParseException {
+ skipWS();
+ TemplateModel result = consumeValue("Empty JSON (contains no value)", p);
+
+ skipWS();
+ if (p != ln) {
+ throw newParseException("End-of-file was expected but found further non-whitespace characters.");
+ }
+
+ return result;
+ }
+
+ private TemplateModel consumeValue(String eofErrorMessage, int eofBlamePosition) throws JSONParseException {
+ if (p == ln) {
+ throw newParseException(
+ eofErrorMessage == null
+ ? "A value was expected here, but end-of-file was reached." : eofErrorMessage,
+ eofBlamePosition == -1 ? p : eofBlamePosition);
+ }
+
+ TemplateModel result;
+
+ result = tryConsumeString();
+ if (result != null) return result;
+
+ result = tryConsumeNumber();
+ if (result != null) return result;
+
+ result = tryConsumeObject();
+ if (result != null) return result;
+
+ result = tryConsumeArray();
+ if (result != null) return result;
+
+ result = tryConsumeTrueFalseNull();
+ if (result != null) return result != TemplateNullModel.INSTANCE ? result : null;
+
+ // Better error message for a frequent mistake:
+ if (p < ln && src.charAt(p) == '\'') {
+ throw newParseException("Unexpected apostrophe-quote character. "
+ + "JSON strings must be quoted with quotation mark.");
+ }
+
+ throw newParseException(
+ "Expected either the beginning of a (negative) number or the beginning of one of these: "
+ + "{...}, [...], \"...\", true, false, null. Found character " + StringUtil.jQuote(src.charAt(p))
+ + " instead.");
+ }
+
+ private TemplateModel tryConsumeTrueFalseNull() throws JSONParseException {
+ int startP = p;
+ if (p < ln && isIdentifierStart(src.charAt(p))) {
+ p++;
+ while (p < ln && isIdentifierPart(src.charAt(p))) {
+ p++;
+ }
+ }
+
+ if (startP == p) return null;
+
+ String keyword = src.substring(startP, p);
+ if (keyword.equals("true")) {
+ return TemplateBooleanModel.TRUE;
+ } else if (keyword.equals("false")) {
+ return TemplateBooleanModel.FALSE;
+ } else if (keyword.equals("null")) {
+ return TemplateNullModel.INSTANCE;
+ }
+
+ throw newParseException(
+ "Invalid JSON keyword: " + StringUtil.jQuote(keyword)
+ + ". Should be one of: true, false, null. "
+ + "If it meant to be a string then it must be quoted.", startP);
+ }
+
+ private TemplateNumberModel tryConsumeNumber() throws JSONParseException {
+ if (p >= ln) {
+ return null;
+ }
+ char c = src.charAt(p);
+ boolean negative = c == '-';
+ if (!(negative || isDigit(c) || c == '.')) {
+ return null;
+ }
+
+ int startP = p;
+
+ if (negative) {
+ if (p + 1 >= ln) {
+ throw newParseException("Expected a digit after \"-\", but reached end-of-file.");
+ }
+ char lookAheadC = src.charAt(p + 1);
+ if (!(isDigit(lookAheadC) || lookAheadC == '.')) {
+ return null;
+ }
+ p++; // Consume "-" only, not the digit
+ }
+
+ long longSum = 0;
+ boolean firstDigit = true;
+ consumeLongFittingHead: do {
+ c = src.charAt(p);
+
+ if (!isDigit(c)) {
+ if (c == '.' && firstDigit) {
+ throw newParseException("JSON doesn't allow numbers starting with \".\".");
+ }
+ break consumeLongFittingHead;
+ }
+
+ int digit = c - '0';
+ if (longSum == 0) {
+ if (!firstDigit) {
+ throw newParseException("JSON doesn't allow superfluous leading 0-s.", p - 1);
+ }
+
+ longSum = !negative ? digit : -digit;
+ p++;
+ } else {
+ long prevLongSum = longSum;
+ longSum = longSum * 10 + (!negative ? digit : -digit);
+ if (!negative && prevLongSum > longSum || negative && prevLongSum < longSum) {
+ // We had an overflow => Can't consume this digit as long-fitting
+ break consumeLongFittingHead;
+ }
+ p++;
+ }
+ firstDigit = false;
+ } while (p < ln);
+
+ if (p < ln && isBigDecimalFittingTailCharacter(c)) {
+ char lastC = c;
+ p++;
+
+ consumeBigDecimalFittingTail: while (p < ln) {
+ c = src.charAt(p);
+ if (isBigDecimalFittingTailCharacter(c)) {
+ p++;
+ } else if ((c == '+' || c == '-') && isE(lastC)) {
+ p++;
+ } else {
+ break consumeBigDecimalFittingTail;
+ }
+ lastC = c;
+ }
+
+ String numStr = src.substring(startP, p);
+ BigDecimal bd;
+ try {
+ bd = new BigDecimal(numStr);
+ } catch (NumberFormatException e) {
+ throw new JSONParseException("Malformed number: " + numStr, src, startP, e);
+ }
+
+ if (bd.compareTo(MIN_INT_AS_BIGDECIMAL) >= 0 && bd.compareTo(MAX_INT_AS_BIGDECIMAL) <= 0) {
+ if (NumberUtil.isIntegerBigDecimal(bd)) {
+ return new SimpleNumber(bd.intValue());
+ }
+ } else if (bd.compareTo(MIN_LONG_AS_BIGDECIMAL) >= 0 && bd.compareTo(MAX_LONG_AS_BIGDECIMAL) <= 0) {
+ if (NumberUtil.isIntegerBigDecimal(bd)) {
+ return new SimpleNumber(bd.longValue());
+ }
+ }
+ return new SimpleNumber(bd);
+ } else {
+ return new SimpleNumber(
+ longSum <= Integer.MAX_VALUE && longSum >= Integer.MIN_VALUE
+ ? (Number) (int) longSum
+ : longSum);
+ }
+ }
+
+ private TemplateScalarModel tryConsumeString() throws JSONParseException {
+ int startP = p;
+ if (!tryConsumeChar('"')) return null;
+
+ StringBuilder sb = new StringBuilder();
+ char c = 0;
+ while (p < ln) {
+ c = src.charAt(p);
+
+ if (c == '"') {
+ p++;
+ return new SimpleScalar(sb.toString()); // Call normally returns here!
+ } else if (c == '\\') {
+ p++;
+ sb.append(consumeAfterBackslash());
+ } else if (c <= 0x1F) {
+ throw newParseException("JSON doesn't allow unescaped control characters in string literals, "
+ + "but found character with code (decimal): " + (int) c);
+ } else {
+ p++;
+ sb.append(c);
+ }
+ }
+
+ throw newParseException("String literal was still unclosed when the end of the file was reached. "
+ + "(Look for missing or accidentally escaped closing quotation mark.)", startP);
+ }
+
+ private TemplateSequenceModel tryConsumeArray() throws JSONParseException {
+ int startP = p;
+ if (!tryConsumeChar('[')) return null;
+
+ skipWS();
+ if (tryConsumeChar(']')) return Constants.EMPTY_SEQUENCE;
+
+ boolean afterComma = false;
+ SimpleSequence elements = new SimpleSequence(_TemplateAPI.SAFE_OBJECT_WRAPPER);
+ do {
+ skipWS();
+ elements.add(consumeValue(afterComma ? null : UNCLOSED_ARRAY_MESSAGE, afterComma ? -1 : startP));
+
+ skipWS();
+ afterComma = true;
+ } while (consumeChar(',', ']', UNCLOSED_ARRAY_MESSAGE, startP) == ',');
+ return elements;
+ }
+
+ private TemplateHashModelEx2 tryConsumeObject() throws JSONParseException {
+ int startP = p;
+ if (!tryConsumeChar('{')) return null;
+
+ skipWS();
+ if (tryConsumeChar('}')) return Constants.EMPTY_HASH_EX2;
+
+ boolean afterComma = false;
+ Map<String, Object> map = new LinkedHashMap<>(); // Must keeps original order!
+ do {
+ skipWS();
+ int keyStartP = p;
+ Object key = consumeValue(afterComma ? null : UNCLOSED_OBJECT_MESSAGE, afterComma ? -1 : startP);
+ if (!(key instanceof TemplateScalarModel)) {
+ throw newParseException("Wrong key type. JSON only allows string keys inside {...}.", keyStartP);
+ }
+ String strKey = null;
+ try {
+ strKey = ((TemplateScalarModel) key).getAsString();
+ } catch (TemplateModelException e) {
+ throw new BugException(e);
+ }
+
+ skipWS();
+ consumeChar(':');
+
+ skipWS();
+ map.put(strKey, consumeValue(null, -1));
+
+ skipWS();
+ afterComma = true;
+ } while (consumeChar(',', '}', UNCLOSED_OBJECT_MESSAGE, startP) == ',');
+ return new SimpleHash(map, _TemplateAPI.SAFE_OBJECT_WRAPPER, 0);
+ }
+
+ private boolean isE(char c) {
+ return c == 'e' || c == 'E';
+ }
+
+ private boolean isBigDecimalFittingTailCharacter(char c) {
+ return c == '.' || isE(c) || isDigit(c);
+ }
+
+ private char consumeAfterBackslash() throws JSONParseException {
+ if (p == ln) {
+ throw newParseException("Reached the end of the file, but the escape is unclosed.");
+ }
+
+ final char c = src.charAt(p);
+ switch (c) {
+ case '"':
+ case '\\':
+ case '/':
+ p++;
+ return c;
+ case 'b':
+ p++;
+ return '\b';
+ case 'f':
+ p++;
+ return '\f';
+ case 'n':
+ p++;
+ return '\n';
+ case 'r':
+ p++;
+ return '\r';
+ case 't':
+ p++;
+ return '\t';
+ case 'u':
+ p++;
+ return consumeAfterBackslashU();
+ }
+ throw newParseException("Unsupported escape: \\" + c);
+ }
+
+ private char consumeAfterBackslashU() throws JSONParseException {
+ if (p + 3 >= ln) {
+ throw newParseException("\\u must be followed by exactly 4 hexadecimal digits");
+ }
+ final String hex = src.substring(p, p + 4);
+ try {
+ char r = (char) Integer.parseInt(hex, 16);
+ p += 4;
+ return r;
+ } catch (NumberFormatException e) {
+ throw newParseException("\\u must be followed by exactly 4 hexadecimal digits, but was followed by "
+ + StringUtil.jQuote(hex) + ".");
+ }
+ }
+
+ private boolean tryConsumeChar(char c) {
+ if (p < ln && src.charAt(p) == c) {
+ p++;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private void consumeChar(char expected) throws JSONParseException {
+ consumeChar(expected, (char) 0, null, -1);
+ }
+
+ private char consumeChar(char expected1, char expected2, String eofErrorHint, int eofErrorP) throws JSONParseException {
+ if (p >= ln) {
+ throw newParseException(eofErrorHint == null
+ ? "Expected " + StringUtil.jQuote(expected1)
+ + ( expected2 != 0 ? " or " + StringUtil.jQuote(expected2) : "")
+ + " character, but reached end-of-file. "
+ : eofErrorHint,
+ eofErrorP == -1 ? p : eofErrorP);
+ }
+ char c = src.charAt(p);
+ if (c == expected1 || (expected2 != 0 && c == expected2)) {
+ p++;
+ return c;
+ }
+ throw newParseException("Expected " + StringUtil.jQuote(expected1)
+ + ( expected2 != 0 ? " or " + StringUtil.jQuote(expected2) : "")
+ + " character, but found " + StringUtil.jQuote(c) + " instead.");
+ }
+
+ private void skipWS() throws JSONParseException {
+ do {
+ while (p < ln && isWS(src.charAt(p))) {
+ p++;
+ }
+ } while (skipComment());
+ }
+
+ private boolean skipComment() throws JSONParseException {
+ if (p + 1 < ln) {
+ if (src.charAt(p) == '/') {
+ char c2 = src.charAt(p + 1);
+ if (c2 == '/') {
+ int eolP = p + 2;
+ while (eolP < ln && !isLineBreak(src.charAt(eolP))) {
+ eolP++;
+ }
+ p = eolP;
+ return true;
+ } else if (c2 == '*') {
+ int closerP = p + 3;
+ while (closerP < ln && !(src.charAt(closerP - 1) == '*' && src.charAt(closerP) == '/')) {
+ closerP++;
+ }
+ if (closerP >= ln) {
+ throw newParseException("Unclosed comment");
+ }
+ p = closerP + 1;
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Whitespace as specified by JSON, plus non-breaking space (nbsp), and BOM.
+ */
+ private static boolean isWS(char c) {
+ return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == 0xA0 || c == '\uFEFF';
+ }
+
+ private static boolean isLineBreak(char c) {
+ return c == '\r' || c == '\n';
+ }
+
+ private static boolean isIdentifierStart(char c) {
+ return Character.isLetter(c) || c == '_' || c == '$';
+ }
+
+ private static boolean isDigit(char c) {
+ return c >= '0' && c <= '9';
+ }
+
+ private static boolean isIdentifierPart(char c) {
+ return isIdentifierStart(c) || isDigit(c);
+ }
+
+ private JSONParseException newParseException(String message) {
+ return newParseException(message, p);
+ }
+
+ private JSONParseException newParseException(String message, int p) {
+ return new JSONParseException(message, src, p);
+ }
+
+ static class JSONParseException extends Exception {
+ public JSONParseException(String message, String src, int position) {
+ super(createSourceCodeErrorMessage(message, src, position));
+ }
+
+ public JSONParseException(String message, String src, int position,
+ Throwable cause) {
+ super(createSourceCodeErrorMessage(message, src, position), cause);
+ }
+
+ }
+
+ private static int MAX_QUOTATION_LENGTH = 50;
+
+ private static String createSourceCodeErrorMessage(String message, String srcCode, int position) {
+ int ln = srcCode.length();
+ if (position < 0) {
+ position = 0;
+ }
+ if (position >= ln) {
+ return message + "\n"
+ + "Error location: At the end of text.";
+ }
+
+ int i;
+ char c;
+ int rowBegin = 0;
+ int rowEnd;
+ int row = 1;
+ char lastChar = 0;
+ for (i = 0; i <= position; i++) {
+ c = srcCode.charAt(i);
+ if (lastChar == 0xA) {
+ rowBegin = i;
+ row++;
+ } else if (lastChar == 0xD && c != 0xA) {
+ rowBegin = i;
+ row++;
+ }
+ lastChar = c;
+ }
+ for (i = position; i < ln; i++) {
+ c = srcCode.charAt(i);
+ if (c == 0xA || c == 0xD) {
+ if (c == 0xA && i > 0 && srcCode.charAt(i - 1) == 0xD) {
+ i--;
+ }
+ break;
+ }
+ }
+ rowEnd = i - 1;
+ if (position > rowEnd + 1) {
+ position = rowEnd + 1;
+ }
+ int col = position - rowBegin + 1;
+ if (rowBegin > rowEnd) {
+ return message + "\n"
+ + "Error location: line "
+ + row + ", column " + col + ":\n"
+ + "(Can't show the line because it is empty.)";
+ }
+ String s1 = srcCode.substring(rowBegin, position);
+ String s2 = srcCode.substring(position, rowEnd + 1);
+ s1 = expandTabs(s1, 8);
+ int ln1 = s1.length();
+ s2 = expandTabs(s2, 8, ln1);
+ int ln2 = s2.length();
+ if (ln1 + ln2 > MAX_QUOTATION_LENGTH) {
+ int newLn2 = ln2 - ((ln1 + ln2) - MAX_QUOTATION_LENGTH);
+ if (newLn2 < 6) {
+ newLn2 = 6;
+ }
+ if (newLn2 < ln2) {
+ s2 = s2.substring(0, newLn2 - 3) + "...";
+ ln2 = newLn2;
+ }
+ if (ln1 + ln2 > MAX_QUOTATION_LENGTH) {
+ s1 = "..." + s1.substring((ln1 + ln2) - MAX_QUOTATION_LENGTH + 3);
+ }
+ }
+ StringBuilder res = new StringBuilder(message.length() + 80);
+ res.append(message);
+ res.append("\nError location: line ").append(row).append(", column ").append(col).append(":\n");
+ res.append(s1).append(s2).append("\n");
+ int x = s1.length();
+ while (x != 0) {
+ res.append(' ');
+ x--;
+ }
+ res.append('^');
+
+ return res.toString();
+ }
+
+ private static String expandTabs(String s, int tabWidth) {
+ return expandTabs(s, tabWidth, 0);
+ }
+
+ /**
+ * Replaces all tab-s with spaces in a single line.
+ */
+ private static String expandTabs(String s, int tabWidth, int startCol) {
+ int e = s.indexOf('\t');
+ if (e == -1) {
+ return s;
+ }
+ int b = 0;
+ StringBuilder buf = new StringBuilder(s.length() + Math.max(16, tabWidth * 2));
+ do {
+ buf.append(s, b, e);
+ int col = buf.length() + startCol;
+ for (int i = tabWidth * (1 + col / tabWidth) - col; i > 0; i--) {
+ buf.append(' ');
+ }
+ b = e + 1;
+ e = s.indexOf('\t', b);
+ } while (e != -1);
+ buf.append(s, b, s.length());
+ return buf.toString();
+ }
+
+}
diff --git a/src/manual/en_US/book.xml b/src/manual/en_US/book.xml
index ffdc017..4d5fba3 100644
--- a/src/manual/en_US/book.xml
+++ b/src/manual/en_US/book.xml
@@ -12795,6 +12795,11 @@ grant codeBase "file:/path/to/freemarker.jar"
</listitem>
<listitem>
+ <para><link
+ linkend="ref_builtin_eval_json">eval_json</link></para>
+ </listitem>
+
+ <listitem>
<para><link linkend="ref_builtin_filter">filter</link></para>
</listitem>
@@ -19168,6 +19173,16 @@ Filer for positives:
linkend="ref_builtin_interpret"><literal>interpret</literal>
built-in</link> instead.)</para>
+ <important>
+ <para>Do not use this to evaluate JSON! For that use the <link
+ linkend="ref_builtin_eval_json"><literal>eval_json</literal>
+ built-in</link> instead. While FTL expression language looks
+ similar to JSON, not all JSON is valid FTL expression. Also, FTL
+ expressions can access variables, and call Java methods on them,
+ so if you <literal>?eval</literal> strings coming from untrusted
+ source, it can become an attack vector.</para>
+ </important>
+
<para>The evaluated expression sees the same variables (such as
locals) that are visible at the place of the invocation of
<literal>eval</literal>. That is, it behaves similarly as if in
@@ -19184,6 +19199,75 @@ Filer for positives:
built-in</link>.</para>
</section>
+ <section xml:id="ref_builtin_eval_json">
+ <title>eval_json</title>
+
+ <indexterm>
+ <primary>eval_json</primary>
+ </indexterm>
+
+ <indexterm>
+ <primary>evaluate string</primary>
+ </indexterm>
+
+ <indexterm>
+ <primary>JSON</primary>
+ </indexterm>
+
+ <note>
+ <para>This built-in is available since FreeMarker 2.3.31.</para>
+ </note>
+
+ <para>This built-in evaluates a string as a JSON
+ <emphasis>expression</emphasis>, so that you can extract data from
+ inside it. For example, if you receive data in the
+ <literal>dataJson</literal> variable, but it's unfortunately just a
+ flat string that contains <literal>{"name": "foo", "ids": [11,
+ 22]}</literal>, then you can extract data from it like this:</para>
+
+ <programlisting role="template"><#assign data = dataJson<emphasis>?eval_json</emphasis>>
+<p>Name: ${data.name}
+<p>Ids:
+<ul>
+ <#list data.ids as id>
+ <li>${id}
+ </#list>
+</ul></programlisting>
+
+ <para>Ideally, you shouldn't need <literal>eval_json</literal>,
+ since the template should receive data already parsed (to
+ <literal>List</literal>-s, <literal>Map</literal>-s, Java beans,
+ etc.). This built-in is there as a workaround, if you can't improve
+ the data-model.</para>
+
+ <para>The evaluated JSON expression doesn't have to be a JSON object
+ (key-value pairs), it can be any kind of JSON value, like JSON
+ array, JSON number, etc.</para>
+
+ <para>The syntax understood by this built-in is a superset of
+ JSON:</para>
+
+ <itemizedlist>
+ <listitem>
+ <para>Java-style comments are supported
+ (<literal>/*<replaceable>...</replaceable>*/</literal> and
+ <literal>//<replaceable>...</replaceable></literal>)</para>
+ </listitem>
+
+ <listitem>
+ <para>BOM (byte order mark) and non-breaking space
+ (<quote>nbsp</quote>) are treated as whitespace (in a stricter
+ JSON parser they are errors of occurring around tokens).</para>
+ </listitem>
+ </itemizedlist>
+
+ <para>No other non-JSON extras are implemented, notably, it's
+ impossible to refer to variables (unlike in the <link
+ linkend="ref_builtin_eval"><literal>eval</literal> built-in</link>).
+ This is important for safety, when receiving JSON from untrusted
+ sources.</para>
+ </section>
+
<section xml:id="ref_builtin_has_content">
<title>has_content</title>
@@ -29317,6 +29401,20 @@ TemplateModel x = env.getVariable("x"); // get variable x</programlisting>
<para>Release date: FIXME</para>
<section>
+ <title>Changes on the FTL side</title>
+
+ <itemizedlist>
+ <listitem>
+ <para>Added <literal>?eval_json</literal> to evaluate JSON given
+ as flat string. This was added as <literal>?eval</literal> is
+ routinely misused for the same purpose, which not only doesn't
+ work for all JSON-s, but can be a security problem. <link
+ linkend="ref_builtin_eval_json">See more here...</link></para>
+ </listitem>
+ </itemizedlist>
+ </section>
+
+ <section>
<title>Changes on the Java side</title>
<itemizedlist>
diff --git a/src/test/java/freemarker/core/EvalJsonBuiltInTest.java b/src/test/java/freemarker/core/EvalJsonBuiltInTest.java
new file mode 100644
index 0000000..ff62a61
--- /dev/null
+++ b/src/test/java/freemarker/core/EvalJsonBuiltInTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package freemarker.core;
+
+import java.io.IOException;
+
+import org.junit.Test;
+
+import freemarker.template.TemplateException;
+import freemarker.test.TemplateTest;
+
+public class EvalJsonBuiltInTest extends TemplateTest {
+
+ @Test
+ public void test() throws Exception {
+ assertOutput("${'1'?eval_json}", "1");
+ assertOutput("${'1'?evalJson}", "1");
+
+ assertOutput("${'null'?evalJson!'-'}", "-");
+
+ assertOutput("<#list '{\"a\": 1e2, \"b\": null}'?evalJson as k, v>${k}=${v!'NULL'}<#sep>, </#list>", "a=100, b=NULL");
+ }
+
+}
diff --git a/src/test/java/freemarker/core/JSONParserTest.java b/src/test/java/freemarker/core/JSONParserTest.java
new file mode 100644
index 0000000..dfdf5b6
--- /dev/null
+++ b/src/test/java/freemarker/core/JSONParserTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package freemarker.core;
+
+import static org.hamcrest.Matchers.*;
+import static org.junit.Assert.*;
+
+import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+import freemarker.template.TemplateModel;
+import freemarker.template.TemplateModelException;
+import freemarker.template.utility.DeepUnwrap;
+
+public class JSONParserTest {
+
+ @Test
+ public void testObjects() throws JSONParser.JSONParseException {
+ assertEquals(ImmutableMap.of("a", 1, "b", 2), JSONParser.parse("{\"a\": 1, \"b\": 2}"));
+ assertEquals(Collections.emptyMap(), JSONParser.parse("{}"));
+ try {
+ JSONParser.parse("{1: 1}");
+ fail();
+ } catch (JSONParser.JSONParseException e) {
+ assertThat(e.getMessage(), containsString("string key"));
+ }
+ }
+
+ @Test
+ public void testLists() throws JSONParser.JSONParseException {
+ assertEquals(ImmutableList.of(1, 2), JSONParser.parse("[1, 2]"));
+ assertEquals(Collections.emptyList(), JSONParser.parse("[]"));
+ }
+
+ @Test
+ public void testStrings() throws JSONParser.JSONParseException {
+ assertEquals("", JSONParser.parse("\"\""));
+ assertEquals(" ", JSONParser.parse("\" \""));
+ assertEquals("'", JSONParser.parse("\"'\""));
+ assertEquals("foo", JSONParser.parse("\"foo\""));
+ assertEquals("\" \\ / \b \f \n \r \t \ufeff",
+ JSONParser.parse(
+ "\"" +
+ "\\\" \\\\ \\/ \\b \\f \\n \\r \\t \\uFEFF" +
+ "\""));
+ }
+
+ @Test
+ public void testNumbers() throws JSONParser.JSONParseException {
+ assertEquals(0, JSONParser.parse("0"));
+ assertEquals(123, JSONParser.parse("123"));
+ assertEquals(-123, JSONParser.parse("-123"));
+ assertNotEquals(123L, JSONParser.parse("123"));
+ assertEquals(2147483647, JSONParser.parse("2147483647"));
+ assertEquals(2147483648L, JSONParser.parse("2147483648"));
+ assertEquals(-2147483648, JSONParser.parse("-2147483648"));
+ assertEquals(-2147483649L, JSONParser.parse("-2147483649"));
+ assertEquals(-123, JSONParser.parse("-1.23E2"));
+ assertEquals(new BigDecimal("1.23"), JSONParser.parse("1.23"));
+ assertEquals(new BigDecimal("-1.23"), JSONParser.parse("-1.23"));
+ assertEquals(new BigDecimal("12.3"), JSONParser.parse("1.23E1"));
+ assertEquals(new BigDecimal("0.123"), JSONParser.parse("123E-3"));
+ }
+
+ @Test
+ public void testKeywords() throws JSONParser.JSONParseException {
+ assertNull(JSONParser.parse("null"));
+ assertEquals(true, JSONParser.parse("true"));
+ assertEquals(false, JSONParser.parse("false"));
+ try {
+ JSONParser.parse("NULL");
+ fail();
+ } catch (JSONParser.JSONParseException e) {
+ assertThat(e.getMessage(), containsString("quoted"));
+ }
+ }
+
+ @Test
+ public void testBlockComments() throws JSONParser.JSONParseException {
+ assertEquals(ImmutableList.of(1, 2), JSONParser.parse("/**/[/**/1/**/, /**/2/**/]/**/"));
+ assertEquals(ImmutableList.of(1, 2), JSONParser.parse("/*x*/[/*x*/1/*x*/, /*x*/2/*x*/]/*x*/"));
+ assertEquals(ImmutableList.of(1), JSONParser.parse(" /*x*/ /**//**/ [ /*x*/ /*\n*//***/ 1 ]"));
+ try {
+ JSONParser.parse("/*");
+ fail();
+ } catch (JSONParser.JSONParseException e) {
+ assertThat(e.getMessage(), containsString("Unclosed comment"));
+ }
+ try {
+ JSONParser.parse("[/*]");
+ fail();
+ } catch (JSONParser.JSONParseException e) {
+ assertThat(e.getMessage(), containsString("Unclosed comment"));
+ }
+ }
+
+ @Test
+ public void testLineComments() throws JSONParser.JSONParseException {
+ assertEquals(ImmutableList.of(1, 2), JSONParser.parse("//c1\n[ //c2\n1, //c3\n 2//c5\n] //c4"));
+ assertEquals(ImmutableList.of(1, 2), JSONParser.parse("// c1\n//\r// c2\r\n// c3\r\n[ 1, 2 ]//"));
+ assertEquals(ImmutableList.of(1, 2), JSONParser.parse("[1, 2]\n//\n"));
+ }
+
+ @Test
+ public void testWhitespace() throws JSONParser.JSONParseException {
+ assertEquals(ImmutableList.of(1, 2), JSONParser.parse(" [ 1 ,\n2 ] "));
+ assertEquals(ImmutableList.of(1, 2), JSONParser.parse("\uFEFF[\u00A01\u00A0,2]"));
+ }
+
+ @Test
+ public void testMixed() throws JSONParser.JSONParseException {
+ LinkedHashMap<String, Object> m = new LinkedHashMap<>();
+ m.put("x", 1);
+ m.put("y", null);
+ assertEquals(
+ ImmutableList.of(
+ ImmutableMap.of("a", Collections.emptyMap()),
+ ImmutableMap.of("b",
+ Arrays.asList(
+ m,
+ true,
+ null
+ ))
+ ),
+ JSONParser.parse("" +
+ "[\n" +
+ "{\"a\":{}},\n" +
+ "{\"b\":\n" +
+ "[" +
+ "{\"x\":1, \"y\": null}," +
+ "true," +
+ "null" +
+ "] // comment\n" +
+ "}\n" +
+ "]"));
+ }
+
+ private static void assertEquals(Object expected, TemplateModel actual) {
+ try {
+ Assert.assertEquals(expected, DeepUnwrap.unwrap(actual));
+ } catch (TemplateModelException e) {
+ throw new BugException(e);
+ }
+ }
+
+}
\ No newline at end of file