You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@freemarker.apache.org by dd...@apache.org on 2020/10/13 19:57:14 UTC
[freemarker] branch 2.3-gae updated (378ebfc -> 11c2b09)
This is an automated email from the ASF dual-hosted git repository.
ddekany pushed a change to branch 2.3-gae
in repository https://gitbox.apache.org/repos/asf/freemarker.git.
from 378ebfc Manual: Updated version history
new 125b8e6 (JavaDoc typo fixes)
new 11c2b09 Added ?eval_json to evaluate JSON given as flat string. This was added as ?eval is routinely misused for the same purpose.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/main/java/freemarker/core/BuiltIn.java | 4 +-
.../freemarker/core/BuiltInsForStringsMisc.java | 17 +-
src/main/java/freemarker/core/JSONParser.java | 622 +++++++++++++++++++++
.../java/freemarker/core/TemplateNullModel.java | 4 +-
.../java/freemarker/template/_TemplateAPI.java | 4 +-
src/manual/en_US/book.xml | 98 ++++
.../EvalJsonBuiltInTest.java} | 15 +-
src/test/java/freemarker/core/JSONParserTest.java | 171 ++++++
8 files changed, 923 insertions(+), 12 deletions(-)
create mode 100644 src/main/java/freemarker/core/JSONParser.java
copy src/test/java/freemarker/{ext/beans/Java9InstrospectorBugWorkaroundTest.java => core/EvalJsonBuiltInTest.java} (70%)
create mode 100644 src/test/java/freemarker/core/JSONParserTest.java
[freemarker] 01/02: (JavaDoc typo fixes)
Posted by dd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ddekany pushed a commit to branch 2.3-gae
in repository https://gitbox.apache.org/repos/asf/freemarker.git
commit 125b8e65a7b451fd55ec06417bdcfc01fb7ef9a7
Author: ddekany <dd...@apache.org>
AuthorDate: Tue Oct 13 17:13:10 2020 +0200
(JavaDoc typo fixes)
---
src/main/java/freemarker/core/TemplateNullModel.java | 4 ++--
src/main/java/freemarker/template/_TemplateAPI.java | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/main/java/freemarker/core/TemplateNullModel.java b/src/main/java/freemarker/core/TemplateNullModel.java
index b3ac0de..b9af584 100644
--- a/src/main/java/freemarker/core/TemplateNullModel.java
+++ b/src/main/java/freemarker/core/TemplateNullModel.java
@@ -24,9 +24,9 @@ import freemarker.template.TemplateModel;
/**
* Represents a {@code null} value; if we get this as the value of a variable from a scope, we do not fall back
* to a higher scope to get the same variable again. If instead we get a {@code null}, that means that the variable
- * doesn't exist at all in the current scope, and so we fall back to a higher scope. This distinction wasn is only
+ * doesn't exist at all in the current scope, and so we fall back to a higher scope. This distinction is only
* used for (and expected from) certain scopes, so be careful where you are using it. (As of this
- * writing, it's onlt for local variables, including loop variables). The user should never meet a
+ * writing, it's only for local variables, including loop variables). The user should never meet a
* {@link TemplateNullModel}, it must not be returned from public API-s.
*
* @see Environment#getNullableLocalVariable(String)
diff --git a/src/main/java/freemarker/template/_TemplateAPI.java b/src/main/java/freemarker/template/_TemplateAPI.java
index 71feecb..5ff74de 100644
--- a/src/main/java/freemarker/template/_TemplateAPI.java
+++ b/src/main/java/freemarker/template/_TemplateAPI.java
@@ -60,13 +60,13 @@ public class _TemplateAPI {
/**
* Kind of a dummy {@link ObjectWrapper} used at places where the internal code earlier used the
* {@link ObjectWrapper#DEFAULT_WRAPPER} singleton, because it wasn't supposed to wrap/unwrap anything with it;
- * never use this {@link ObjectWrapper}r in situations where values of arbitrary types need to be wrapped!
+ * never use this {@link ObjectWrapper} in situations where values of arbitrary types need to be wrapped!
* The typical situation is that we are using {@link SimpleSequence}, or {@link SimpleHash}, which always has an
* {@link ObjectWrapper} field, even if we don't care in the given situation, and so we didn't set it explicitly.
* The concern with the old way is that the {@link ObjectWrapper} set in the {@link Configuration} is possibly
* more restrictive than the default, so if the template author can somehow make FreeMarker wrap something with the
* default {@link ObjectWrapper}, then we got a security problem. So we try not to have that around, if possible.
- * The obvious fix, and the better engineering would be just use a such {@link TemplateSequenceModel} or
+ * The obvious fix, and the better engineering would be just use a {@link TemplateSequenceModel} or
* {@link TemplateHashModelEx2} implementation at those places, which doesn't have an {@link ObjectWrapper} (and
* doesn't have the overhead of said implementations either). But, some user code might casts the values it
* receives (as directive argument for example) to {@link SimpleSequence} or {@link SimpleHash}, instead of to
[freemarker] 02/02: Added ?eval_json to evaluate JSON given as flat
string. This was added as ?eval is routinely misused for the same purpose.
Posted by dd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ddekany pushed a commit to branch 2.3-gae
in repository https://gitbox.apache.org/repos/asf/freemarker.git
commit 11c2b09574899f1779ad4941adc1994455d53828
Author: ddekany <dd...@apache.org>
AuthorDate: Tue Oct 13 17:13:39 2020 +0200
Added ?eval_json to evaluate JSON given as flat string. This was added as ?eval is routinely misused for the same purpose.
---
src/main/java/freemarker/core/BuiltIn.java | 4 +-
.../freemarker/core/BuiltInsForStringsMisc.java | 17 +-
src/main/java/freemarker/core/JSONParser.java | 622 +++++++++++++++++++++
src/manual/en_US/book.xml | 98 ++++
.../java/freemarker/core/EvalJsonBuiltInTest.java | 41 ++
src/test/java/freemarker/core/JSONParserTest.java | 171 ++++++
6 files changed, 951 insertions(+), 2 deletions(-)
diff --git a/src/main/java/freemarker/core/BuiltIn.java b/src/main/java/freemarker/core/BuiltIn.java
index 30be937..fcea193 100644
--- a/src/main/java/freemarker/core/BuiltIn.java
+++ b/src/main/java/freemarker/core/BuiltIn.java
@@ -65,6 +65,7 @@ import freemarker.core.BuiltInsForSequences.sequenceBI;
import freemarker.core.BuiltInsForSequences.sortBI;
import freemarker.core.BuiltInsForSequences.sort_byBI;
import freemarker.core.BuiltInsForStringsMisc.evalBI;
+import freemarker.core.BuiltInsForStringsMisc.evalJsonBI;
import freemarker.template.Configuration;
import freemarker.template.TemplateDateModel;
import freemarker.template.TemplateModel;
@@ -84,7 +85,7 @@ abstract class BuiltIn extends Expression implements Cloneable {
static final Set<String> CAMEL_CASE_NAMES = new TreeSet<>();
static final Set<String> SNAKE_CASE_NAMES = new TreeSet<>();
- static final int NUMBER_OF_BIS = 289;
+ static final int NUMBER_OF_BIS = 291;
static final HashMap<String, BuiltIn> BUILT_INS_BY_NAME = new HashMap(NUMBER_OF_BIS * 3 / 2 + 1, 1f);
static final String BI_NAME_SNAKE_CASE_WITH_ARGS = "with_args";
@@ -120,6 +121,7 @@ abstract class BuiltIn extends Expression implements Cloneable {
putBI("ensure_starts_with", "ensureStartsWith", new BuiltInsForStringsBasic.ensure_starts_withBI());
putBI("esc", new escBI());
putBI("eval", new evalBI());
+ putBI("eval_json", "evalJson", new evalJsonBI());
putBI("exists", new BuiltInsForExistenceHandling.existsBI());
putBI("filter", new BuiltInsForSequences.filterBI());
putBI("first", new firstBI());
diff --git a/src/main/java/freemarker/core/BuiltInsForStringsMisc.java b/src/main/java/freemarker/core/BuiltInsForStringsMisc.java
index 012a007..284abee 100644
--- a/src/main/java/freemarker/core/BuiltInsForStringsMisc.java
+++ b/src/main/java/freemarker/core/BuiltInsForStringsMisc.java
@@ -113,6 +113,22 @@ class BuiltInsForStringsMisc {
}
+ static class evalJsonBI extends BuiltInForString {
+ @Override
+ TemplateModel calculateResult(String s, Environment env) throws TemplateException {
+ try {
+ return JSONParser.parse(s);
+ } catch (JSONParser.JSONParseException e) {
+ throw new _MiscTemplateException(this, env,
+ "Failed to \"?", key, "\" string with this error:\n\n",
+ _MessageUtil.EMBEDDED_MESSAGE_BEGIN,
+ new _DelayedGetMessage(e),
+ _MessageUtil.EMBEDDED_MESSAGE_END,
+ "\n\nThe failing expression:");
+ }
+ }
+ }
+
static class numberBI extends BuiltInForString {
@Override
TemplateModel calculateResult(String s, Environment env) throws TemplateException {
@@ -170,5 +186,4 @@ class BuiltInsForStringsMisc {
// Can't be instantiated
private BuiltInsForStringsMisc() { }
-
}
diff --git a/src/main/java/freemarker/core/JSONParser.java b/src/main/java/freemarker/core/JSONParser.java
new file mode 100644
index 0000000..ddb01c0
--- /dev/null
+++ b/src/main/java/freemarker/core/JSONParser.java
@@ -0,0 +1,622 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package freemarker.core;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import freemarker.template.SimpleHash;
+import freemarker.template.SimpleNumber;
+import freemarker.template.SimpleScalar;
+import freemarker.template.SimpleSequence;
+import freemarker.template.Template;
+import freemarker.template.TemplateBooleanModel;
+import freemarker.template.TemplateHashModelEx2;
+import freemarker.template.TemplateModel;
+import freemarker.template.TemplateModelException;
+import freemarker.template.TemplateNumberModel;
+import freemarker.template.TemplateScalarModel;
+import freemarker.template.TemplateSequenceModel;
+import freemarker.template._TemplateAPI;
+import freemarker.template.utility.Constants;
+import freemarker.template.utility.NumberUtil;
+import freemarker.template.utility.StringUtil;
+
+/**
+ * JSON parser that returns a {@link TameplatModel}, similar to what FTL literals product (and so, what
+ * @code ?eval} would return). A notable difference compared to the result FTL literals is that this doesn't use the
+ * {@link ParserConfiguration#getArithmeticEngine()} to parse numbers, as JSON has its own fixed number syntax. For
+ * numbers this parser returns {@link SimpleNumberModel}-s, where the wrapped numbers will be {@link Integer}-s when
+ * they fit into that, otherwise they will be {@link Long}-s if they fit into that, otherwise they will be
+ * {@link BigDecimal}-s. Another difference to the result of FTL literals is that instead of
+ * {@code HashLiteral.SequenceHash} it uses {@link SimpleHash} with {@link LinkedHashMap} as backing store, for
+ * efficiency.
+ *
+ * <p>This parser allows certain things that are errors in pure JSON:
+ * <ul>
+ * <li>JavaScript comments are supported</li>
+ * <li>Non-breaking space (nbsp) and BOM are treated as whitespace</li>
+ * </ul>
+ */
+class JSONParser {
+
+ private static final String UNCLOSED_OBJECT_MESSAGE
+ = "This {...} was still unclosed when the end of the file was reached. (Look for a missing \"}\")";
+
+ private static final String UNCLOSED_ARRAY_MESSAGE
+ = "This [...] was still unclosed when the end of the file was reached. (Look for a missing \"]\")";
+
+ private static final BigDecimal MIN_INT_AS_BIGDECIMAL = BigDecimal.valueOf(Integer.MIN_VALUE);
+ private static final BigDecimal MAX_INT_AS_BIGDECIMAL = BigDecimal.valueOf(Integer.MAX_VALUE);
+ private static final BigDecimal MIN_LONG_AS_BIGDECIMAL = BigDecimal.valueOf(Long.MIN_VALUE);
+ private static final BigDecimal MAX_LONG_AS_BIGDECIMAL = BigDecimal.valueOf(Long.MAX_VALUE);
+
+ private final String src;
+ private final int ln;
+
+ private int p;
+
+ public static TemplateModel parse(String src) throws JSONParseException {
+ return new JSONParser(src).parse();
+ }
+
+ /**
+ * @param sourceLocation Only used in error messages, maybe {@code null}.
+ */
+ private JSONParser(String src) {
+ this.src = src;
+ this.ln = src.length();
+ }
+
+ private TemplateModel parse() throws JSONParseException {
+ skipWS();
+ TemplateModel result = consumeValue("Empty JSON (contains no value)", p);
+
+ skipWS();
+ if (p != ln) {
+ throw newParseException("End-of-file was expected but found further non-whitespace characters.");
+ }
+
+ return result;
+ }
+
+ private TemplateModel consumeValue(String eofErrorMessage, int eofBlamePosition) throws JSONParseException {
+ if (p == ln) {
+ throw newParseException(
+ eofErrorMessage == null
+ ? "A value was expected here, but end-of-file was reached." : eofErrorMessage,
+ eofBlamePosition == -1 ? p : eofBlamePosition);
+ }
+
+ TemplateModel result;
+
+ result = tryConsumeString();
+ if (result != null) return result;
+
+ result = tryConsumeNumber();
+ if (result != null) return result;
+
+ result = tryConsumeObject();
+ if (result != null) return result;
+
+ result = tryConsumeArray();
+ if (result != null) return result;
+
+ result = tryConsumeTrueFalseNull();
+ if (result != null) return result != TemplateNullModel.INSTANCE ? result : null;
+
+ // Better error message for a frequent mistake:
+ if (p < ln && src.charAt(p) == '\'') {
+ throw newParseException("Unexpected apostrophe-quote character. "
+ + "JSON strings must be quoted with quotation mark.");
+ }
+
+ throw newParseException(
+ "Expected either the beginning of a (negative) number or the beginning of one of these: "
+ + "{...}, [...], \"...\", true, false, null. Found character " + StringUtil.jQuote(src.charAt(p))
+ + " instead.");
+ }
+
+ private TemplateModel tryConsumeTrueFalseNull() throws JSONParseException {
+ int startP = p;
+ if (p < ln && isIdentifierStart(src.charAt(p))) {
+ p++;
+ while (p < ln && isIdentifierPart(src.charAt(p))) {
+ p++;
+ }
+ }
+
+ if (startP == p) return null;
+
+ String keyword = src.substring(startP, p);
+ if (keyword.equals("true")) {
+ return TemplateBooleanModel.TRUE;
+ } else if (keyword.equals("false")) {
+ return TemplateBooleanModel.FALSE;
+ } else if (keyword.equals("null")) {
+ return TemplateNullModel.INSTANCE;
+ }
+
+ throw newParseException(
+ "Invalid JSON keyword: " + StringUtil.jQuote(keyword)
+ + ". Should be one of: true, false, null. "
+ + "If it meant to be a string then it must be quoted.", startP);
+ }
+
+ private TemplateNumberModel tryConsumeNumber() throws JSONParseException {
+ if (p >= ln) {
+ return null;
+ }
+ char c = src.charAt(p);
+ boolean negative = c == '-';
+ if (!(negative || isDigit(c) || c == '.')) {
+ return null;
+ }
+
+ int startP = p;
+
+ if (negative) {
+ if (p + 1 >= ln) {
+ throw newParseException("Expected a digit after \"-\", but reached end-of-file.");
+ }
+ char lookAheadC = src.charAt(p + 1);
+ if (!(isDigit(lookAheadC) || lookAheadC == '.')) {
+ return null;
+ }
+ p++; // Consume "-" only, not the digit
+ }
+
+ long longSum = 0;
+ boolean firstDigit = true;
+ consumeLongFittingHead: do {
+ c = src.charAt(p);
+
+ if (!isDigit(c)) {
+ if (c == '.' && firstDigit) {
+ throw newParseException("JSON doesn't allow numbers starting with \".\".");
+ }
+ break consumeLongFittingHead;
+ }
+
+ int digit = c - '0';
+ if (longSum == 0) {
+ if (!firstDigit) {
+ throw newParseException("JSON doesn't allow superfluous leading 0-s.", p - 1);
+ }
+
+ longSum = !negative ? digit : -digit;
+ p++;
+ } else {
+ long prevLongSum = longSum;
+ longSum = longSum * 10 + (!negative ? digit : -digit);
+ if (!negative && prevLongSum > longSum || negative && prevLongSum < longSum) {
+ // We had an overflow => Can't consume this digit as long-fitting
+ break consumeLongFittingHead;
+ }
+ p++;
+ }
+ firstDigit = false;
+ } while (p < ln);
+
+ if (p < ln && isBigDecimalFittingTailCharacter(c)) {
+ char lastC = c;
+ p++;
+
+ consumeBigDecimalFittingTail: while (p < ln) {
+ c = src.charAt(p);
+ if (isBigDecimalFittingTailCharacter(c)) {
+ p++;
+ } else if ((c == '+' || c == '-') && isE(lastC)) {
+ p++;
+ } else {
+ break consumeBigDecimalFittingTail;
+ }
+ lastC = c;
+ }
+
+ String numStr = src.substring(startP, p);
+ BigDecimal bd;
+ try {
+ bd = new BigDecimal(numStr);
+ } catch (NumberFormatException e) {
+ throw new JSONParseException("Malformed number: " + numStr, src, startP, e);
+ }
+
+ if (bd.compareTo(MIN_INT_AS_BIGDECIMAL) >= 0 && bd.compareTo(MAX_INT_AS_BIGDECIMAL) <= 0) {
+ if (NumberUtil.isIntegerBigDecimal(bd)) {
+ return new SimpleNumber(bd.intValue());
+ }
+ } else if (bd.compareTo(MIN_LONG_AS_BIGDECIMAL) >= 0 && bd.compareTo(MAX_LONG_AS_BIGDECIMAL) <= 0) {
+ if (NumberUtil.isIntegerBigDecimal(bd)) {
+ return new SimpleNumber(bd.longValue());
+ }
+ }
+ return new SimpleNumber(bd);
+ } else {
+ return new SimpleNumber(
+ longSum <= Integer.MAX_VALUE && longSum >= Integer.MIN_VALUE
+ ? (Number) (int) longSum
+ : longSum);
+ }
+ }
+
+ private TemplateScalarModel tryConsumeString() throws JSONParseException {
+ int startP = p;
+ if (!tryConsumeChar('"')) return null;
+
+ StringBuilder sb = new StringBuilder();
+ char c = 0;
+ while (p < ln) {
+ c = src.charAt(p);
+
+ if (c == '"') {
+ p++;
+ return new SimpleScalar(sb.toString()); // Call normally returns here!
+ } else if (c == '\\') {
+ p++;
+ sb.append(consumeAfterBackslash());
+ } else if (c <= 0x1F) {
+ throw newParseException("JSON doesn't allow unescaped control characters in string literals, "
+ + "but found character with code (decimal): " + (int) c);
+ } else {
+ p++;
+ sb.append(c);
+ }
+ }
+
+ throw newParseException("String literal was still unclosed when the end of the file was reached. "
+ + "(Look for missing or accidentally escaped closing quotation mark.)", startP);
+ }
+
+ private TemplateSequenceModel tryConsumeArray() throws JSONParseException {
+ int startP = p;
+ if (!tryConsumeChar('[')) return null;
+
+ skipWS();
+ if (tryConsumeChar(']')) return Constants.EMPTY_SEQUENCE;
+
+ boolean afterComma = false;
+ SimpleSequence elements = new SimpleSequence(_TemplateAPI.SAFE_OBJECT_WRAPPER);
+ do {
+ skipWS();
+ elements.add(consumeValue(afterComma ? null : UNCLOSED_ARRAY_MESSAGE, afterComma ? -1 : startP));
+
+ skipWS();
+ afterComma = true;
+ } while (consumeChar(',', ']', UNCLOSED_ARRAY_MESSAGE, startP) == ',');
+ return elements;
+ }
+
+ private TemplateHashModelEx2 tryConsumeObject() throws JSONParseException {
+ int startP = p;
+ if (!tryConsumeChar('{')) return null;
+
+ skipWS();
+ if (tryConsumeChar('}')) return Constants.EMPTY_HASH_EX2;
+
+ boolean afterComma = false;
+ Map<String, Object> map = new LinkedHashMap<>(); // Must keeps original order!
+ do {
+ skipWS();
+ int keyStartP = p;
+ Object key = consumeValue(afterComma ? null : UNCLOSED_OBJECT_MESSAGE, afterComma ? -1 : startP);
+ if (!(key instanceof TemplateScalarModel)) {
+ throw newParseException("Wrong key type. JSON only allows string keys inside {...}.", keyStartP);
+ }
+ String strKey = null;
+ try {
+ strKey = ((TemplateScalarModel) key).getAsString();
+ } catch (TemplateModelException e) {
+ throw new BugException(e);
+ }
+
+ skipWS();
+ consumeChar(':');
+
+ skipWS();
+ map.put(strKey, consumeValue(null, -1));
+
+ skipWS();
+ afterComma = true;
+ } while (consumeChar(',', '}', UNCLOSED_OBJECT_MESSAGE, startP) == ',');
+ return new SimpleHash(map, _TemplateAPI.SAFE_OBJECT_WRAPPER, 0);
+ }
+
+ private boolean isE(char c) {
+ return c == 'e' || c == 'E';
+ }
+
+ private boolean isBigDecimalFittingTailCharacter(char c) {
+ return c == '.' || isE(c) || isDigit(c);
+ }
+
+ private char consumeAfterBackslash() throws JSONParseException {
+ if (p == ln) {
+ throw newParseException("Reached the end of the file, but the escape is unclosed.");
+ }
+
+ final char c = src.charAt(p);
+ switch (c) {
+ case '"':
+ case '\\':
+ case '/':
+ p++;
+ return c;
+ case 'b':
+ p++;
+ return '\b';
+ case 'f':
+ p++;
+ return '\f';
+ case 'n':
+ p++;
+ return '\n';
+ case 'r':
+ p++;
+ return '\r';
+ case 't':
+ p++;
+ return '\t';
+ case 'u':
+ p++;
+ return consumeAfterBackslashU();
+ }
+ throw newParseException("Unsupported escape: \\" + c);
+ }
+
+ private char consumeAfterBackslashU() throws JSONParseException {
+ if (p + 3 >= ln) {
+ throw newParseException("\\u must be followed by exactly 4 hexadecimal digits");
+ }
+ final String hex = src.substring(p, p + 4);
+ try {
+ char r = (char) Integer.parseInt(hex, 16);
+ p += 4;
+ return r;
+ } catch (NumberFormatException e) {
+ throw newParseException("\\u must be followed by exactly 4 hexadecimal digits, but was followed by "
+ + StringUtil.jQuote(hex) + ".");
+ }
+ }
+
+ private boolean tryConsumeChar(char c) {
+ if (p < ln && src.charAt(p) == c) {
+ p++;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private void consumeChar(char expected) throws JSONParseException {
+ consumeChar(expected, (char) 0, null, -1);
+ }
+
+ private char consumeChar(char expected1, char expected2, String eofErrorHint, int eofErrorP) throws JSONParseException {
+ if (p >= ln) {
+ throw newParseException(eofErrorHint == null
+ ? "Expected " + StringUtil.jQuote(expected1)
+ + ( expected2 != 0 ? " or " + StringUtil.jQuote(expected2) : "")
+ + " character, but reached end-of-file. "
+ : eofErrorHint,
+ eofErrorP == -1 ? p : eofErrorP);
+ }
+ char c = src.charAt(p);
+ if (c == expected1 || (expected2 != 0 && c == expected2)) {
+ p++;
+ return c;
+ }
+ throw newParseException("Expected " + StringUtil.jQuote(expected1)
+ + ( expected2 != 0 ? " or " + StringUtil.jQuote(expected2) : "")
+ + " character, but found " + StringUtil.jQuote(c) + " instead.");
+ }
+
+ private void skipWS() throws JSONParseException {
+ do {
+ while (p < ln && isWS(src.charAt(p))) {
+ p++;
+ }
+ } while (skipComment());
+ }
+
+ private boolean skipComment() throws JSONParseException {
+ if (p + 1 < ln) {
+ if (src.charAt(p) == '/') {
+ char c2 = src.charAt(p + 1);
+ if (c2 == '/') {
+ int eolP = p + 2;
+ while (eolP < ln && !isLineBreak(src.charAt(eolP))) {
+ eolP++;
+ }
+ p = eolP;
+ return true;
+ } else if (c2 == '*') {
+ int closerP = p + 3;
+ while (closerP < ln && !(src.charAt(closerP - 1) == '*' && src.charAt(closerP) == '/')) {
+ closerP++;
+ }
+ if (closerP >= ln) {
+ throw newParseException("Unclosed comment");
+ }
+ p = closerP + 1;
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Whitespace as specified by JSON, plus non-breaking space (nbsp), and BOM.
+ */
+ private static boolean isWS(char c) {
+ return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == 0xA0 || c == '\uFEFF';
+ }
+
+ private static boolean isLineBreak(char c) {
+ return c == '\r' || c == '\n';
+ }
+
+ private static boolean isIdentifierStart(char c) {
+ return Character.isLetter(c) || c == '_' || c == '$';
+ }
+
+ private static boolean isDigit(char c) {
+ return c >= '0' && c <= '9';
+ }
+
+ private static boolean isIdentifierPart(char c) {
+ return isIdentifierStart(c) || isDigit(c);
+ }
+
+ private JSONParseException newParseException(String message) {
+ return newParseException(message, p);
+ }
+
+ private JSONParseException newParseException(String message, int p) {
+ return new JSONParseException(message, src, p);
+ }
+
+ static class JSONParseException extends Exception {
+ public JSONParseException(String message, String src, int position) {
+ super(createSourceCodeErrorMessage(message, src, position));
+ }
+
+ public JSONParseException(String message, String src, int position,
+ Throwable cause) {
+ super(createSourceCodeErrorMessage(message, src, position), cause);
+ }
+
+ }
+
+ private static int MAX_QUOTATION_LENGTH = 50;
+
+ private static String createSourceCodeErrorMessage(String message, String srcCode, int position) {
+ int ln = srcCode.length();
+ if (position < 0) {
+ position = 0;
+ }
+ if (position >= ln) {
+ return message + "\n"
+ + "Error location: At the end of text.";
+ }
+
+ int i;
+ char c;
+ int rowBegin = 0;
+ int rowEnd;
+ int row = 1;
+ char lastChar = 0;
+ for (i = 0; i <= position; i++) {
+ c = srcCode.charAt(i);
+ if (lastChar == 0xA) {
+ rowBegin = i;
+ row++;
+ } else if (lastChar == 0xD && c != 0xA) {
+ rowBegin = i;
+ row++;
+ }
+ lastChar = c;
+ }
+ for (i = position; i < ln; i++) {
+ c = srcCode.charAt(i);
+ if (c == 0xA || c == 0xD) {
+ if (c == 0xA && i > 0 && srcCode.charAt(i - 1) == 0xD) {
+ i--;
+ }
+ break;
+ }
+ }
+ rowEnd = i - 1;
+ if (position > rowEnd + 1) {
+ position = rowEnd + 1;
+ }
+ int col = position - rowBegin + 1;
+ if (rowBegin > rowEnd) {
+ return message + "\n"
+ + "Error location: line "
+ + row + ", column " + col + ":\n"
+ + "(Can't show the line because it is empty.)";
+ }
+ String s1 = srcCode.substring(rowBegin, position);
+ String s2 = srcCode.substring(position, rowEnd + 1);
+ s1 = expandTabs(s1, 8);
+ int ln1 = s1.length();
+ s2 = expandTabs(s2, 8, ln1);
+ int ln2 = s2.length();
+ if (ln1 + ln2 > MAX_QUOTATION_LENGTH) {
+ int newLn2 = ln2 - ((ln1 + ln2) - MAX_QUOTATION_LENGTH);
+ if (newLn2 < 6) {
+ newLn2 = 6;
+ }
+ if (newLn2 < ln2) {
+ s2 = s2.substring(0, newLn2 - 3) + "...";
+ ln2 = newLn2;
+ }
+ if (ln1 + ln2 > MAX_QUOTATION_LENGTH) {
+ s1 = "..." + s1.substring((ln1 + ln2) - MAX_QUOTATION_LENGTH + 3);
+ }
+ }
+ StringBuilder res = new StringBuilder(message.length() + 80);
+ res.append(message);
+ res.append("\nError location: line ").append(row).append(", column ").append(col).append(":\n");
+ res.append(s1).append(s2).append("\n");
+ int x = s1.length();
+ while (x != 0) {
+ res.append(' ');
+ x--;
+ }
+ res.append('^');
+
+ return res.toString();
+ }
+
+ private static String expandTabs(String s, int tabWidth) {
+ return expandTabs(s, tabWidth, 0);
+ }
+
+ /**
+ * Replaces all tab-s with spaces in a single line.
+ */
+ private static String expandTabs(String s, int tabWidth, int startCol) {
+ int e = s.indexOf('\t');
+ if (e == -1) {
+ return s;
+ }
+ int b = 0;
+ StringBuilder buf = new StringBuilder(s.length() + Math.max(16, tabWidth * 2));
+ do {
+ buf.append(s, b, e);
+ int col = buf.length() + startCol;
+ for (int i = tabWidth * (1 + col / tabWidth) - col; i > 0; i--) {
+ buf.append(' ');
+ }
+ b = e + 1;
+ e = s.indexOf('\t', b);
+ } while (e != -1);
+ buf.append(s, b, s.length());
+ return buf.toString();
+ }
+
+}
diff --git a/src/manual/en_US/book.xml b/src/manual/en_US/book.xml
index ffdc017..4d5fba3 100644
--- a/src/manual/en_US/book.xml
+++ b/src/manual/en_US/book.xml
@@ -12795,6 +12795,11 @@ grant codeBase "file:/path/to/freemarker.jar"
</listitem>
<listitem>
+ <para><link
+ linkend="ref_builtin_eval_json">eval_json</link></para>
+ </listitem>
+
+ <listitem>
<para><link linkend="ref_builtin_filter">filter</link></para>
</listitem>
@@ -19168,6 +19173,16 @@ Filer for positives:
linkend="ref_builtin_interpret"><literal>interpret</literal>
built-in</link> instead.)</para>
+ <important>
+ <para>Do not use this to evaluate JSON! For that use the <link
+ linkend="ref_builtin_eval_json"><literal>eval_json</literal>
+ built-in</link> instead. While FTL expression language looks
+ similar to JSON, not all JSON is valid FTL expression. Also, FTL
+ expressions can access variables, and call Java methods on them,
+ so if you <literal>?eval</literal> strings coming from untrusted
+ source, it can become an attack vector.</para>
+ </important>
+
<para>The evaluated expression sees the same variables (such as
locals) that are visible at the place of the invocation of
<literal>eval</literal>. That is, it behaves similarly as if in
@@ -19184,6 +19199,75 @@ Filer for positives:
built-in</link>.</para>
</section>
+ <section xml:id="ref_builtin_eval_json">
+ <title>eval_json</title>
+
+ <indexterm>
+ <primary>eval_json</primary>
+ </indexterm>
+
+ <indexterm>
+ <primary>evaluate string</primary>
+ </indexterm>
+
+ <indexterm>
+ <primary>JSON</primary>
+ </indexterm>
+
+ <note>
+ <para>This built-in is available since FreeMarker 2.3.31.</para>
+ </note>
+
+ <para>This built-in evaluates a string as a JSON
+ <emphasis>expression</emphasis>, so that you can extract data from
+ inside it. For example, if you receive data in the
+ <literal>dataJson</literal> variable, but it's unfortunately just a
+ flat string that contains <literal>{"name": "foo", "ids": [11,
+ 22]}</literal>, then you can extract data from it like this:</para>
+
+ <programlisting role="template"><#assign data = dataJson<emphasis>?eval_json</emphasis>>
+<p>Name: ${data.name}
+<p>Ids:
+<ul>
+ <#list data.ids as id>
+ <li>${id}
+ </#list>
+</ul></programlisting>
+
+ <para>Ideally, you shouldn't need <literal>eval_json</literal>,
+ since the template should receive data already parsed (to
+ <literal>List</literal>-s, <literal>Map</literal>-s, Java beans,
+ etc.). This built-in is there as a workaround, if you can't improve
+ the data-model.</para>
+
+ <para>The evaluated JSON expression doesn't have to be a JSON object
+ (key-value pairs), it can be any kind of JSON value, like JSON
+ array, JSON number, etc.</para>
+
+ <para>The syntax understood by this built-in is a superset of
+ JSON:</para>
+
+ <itemizedlist>
+ <listitem>
+ <para>Java-style comments are supported
+ (<literal>/*<replaceable>...</replaceable>*/</literal> and
+ <literal>//<replaceable>...</replaceable></literal>)</para>
+ </listitem>
+
+ <listitem>
+ <para>BOM (byte order mark) and non-breaking space
+ (<quote>nbsp</quote>) are treated as whitespace (in a stricter
+ JSON parser they are errors of occurring around tokens).</para>
+ </listitem>
+ </itemizedlist>
+
+ <para>No other non-JSON extras are implemented, notably, it's
+ impossible to refer to variables (unlike in the <link
+ linkend="ref_builtin_eval"><literal>eval</literal> built-in</link>).
+ This is important for safety, when receiving JSON from untrusted
+ sources.</para>
+ </section>
+
<section xml:id="ref_builtin_has_content">
<title>has_content</title>
@@ -29317,6 +29401,20 @@ TemplateModel x = env.getVariable("x"); // get variable x</programlisting>
<para>Release date: FIXME</para>
<section>
+ <title>Changes on the FTL side</title>
+
+ <itemizedlist>
+ <listitem>
+ <para>Added <literal>?eval_json</literal> to evaluate JSON given
+ as flat string. This was added as <literal>?eval</literal> is
+ routinely misused for the same purpose, which not only doesn't
+ work for all JSON-s, but can be a security problem. <link
+ linkend="ref_builtin_eval_json">See more here...</link></para>
+ </listitem>
+ </itemizedlist>
+ </section>
+
+ <section>
<title>Changes on the Java side</title>
<itemizedlist>
diff --git a/src/test/java/freemarker/core/EvalJsonBuiltInTest.java b/src/test/java/freemarker/core/EvalJsonBuiltInTest.java
new file mode 100644
index 0000000..ff62a61
--- /dev/null
+++ b/src/test/java/freemarker/core/EvalJsonBuiltInTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package freemarker.core;
+
+import java.io.IOException;
+
+import org.junit.Test;
+
+import freemarker.template.TemplateException;
+import freemarker.test.TemplateTest;
+
+public class EvalJsonBuiltInTest extends TemplateTest {
+
+ @Test
+ public void test() throws Exception {
+ assertOutput("${'1'?eval_json}", "1");
+ assertOutput("${'1'?evalJson}", "1");
+
+ assertOutput("${'null'?evalJson!'-'}", "-");
+
+ assertOutput("<#list '{\"a\": 1e2, \"b\": null}'?evalJson as k, v>${k}=${v!'NULL'}<#sep>, </#list>", "a=100, b=NULL");
+ }
+
+}
diff --git a/src/test/java/freemarker/core/JSONParserTest.java b/src/test/java/freemarker/core/JSONParserTest.java
new file mode 100644
index 0000000..dfdf5b6
--- /dev/null
+++ b/src/test/java/freemarker/core/JSONParserTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package freemarker.core;
+
+import static org.hamcrest.Matchers.*;
+import static org.junit.Assert.*;
+
+import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+import freemarker.template.TemplateModel;
+import freemarker.template.TemplateModelException;
+import freemarker.template.utility.DeepUnwrap;
+
+public class JSONParserTest {
+
+ @Test
+ public void testObjects() throws JSONParser.JSONParseException {
+ assertEquals(ImmutableMap.of("a", 1, "b", 2), JSONParser.parse("{\"a\": 1, \"b\": 2}"));
+ assertEquals(Collections.emptyMap(), JSONParser.parse("{}"));
+ try {
+ JSONParser.parse("{1: 1}");
+ fail();
+ } catch (JSONParser.JSONParseException e) {
+ assertThat(e.getMessage(), containsString("string key"));
+ }
+ }
+
+ @Test
+ public void testLists() throws JSONParser.JSONParseException {
+ assertEquals(ImmutableList.of(1, 2), JSONParser.parse("[1, 2]"));
+ assertEquals(Collections.emptyList(), JSONParser.parse("[]"));
+ }
+
+ @Test
+ public void testStrings() throws JSONParser.JSONParseException {
+ assertEquals("", JSONParser.parse("\"\""));
+ assertEquals(" ", JSONParser.parse("\" \""));
+ assertEquals("'", JSONParser.parse("\"'\""));
+ assertEquals("foo", JSONParser.parse("\"foo\""));
+ assertEquals("\" \\ / \b \f \n \r \t \ufeff",
+ JSONParser.parse(
+ "\"" +
+ "\\\" \\\\ \\/ \\b \\f \\n \\r \\t \\uFEFF" +
+ "\""));
+ }
+
+ @Test
+ public void testNumbers() throws JSONParser.JSONParseException {
+ assertEquals(0, JSONParser.parse("0"));
+ assertEquals(123, JSONParser.parse("123"));
+ assertEquals(-123, JSONParser.parse("-123"));
+ assertNotEquals(123L, JSONParser.parse("123"));
+ assertEquals(2147483647, JSONParser.parse("2147483647"));
+ assertEquals(2147483648L, JSONParser.parse("2147483648"));
+ assertEquals(-2147483648, JSONParser.parse("-2147483648"));
+ assertEquals(-2147483649L, JSONParser.parse("-2147483649"));
+ assertEquals(-123, JSONParser.parse("-1.23E2"));
+ assertEquals(new BigDecimal("1.23"), JSONParser.parse("1.23"));
+ assertEquals(new BigDecimal("-1.23"), JSONParser.parse("-1.23"));
+ assertEquals(new BigDecimal("12.3"), JSONParser.parse("1.23E1"));
+ assertEquals(new BigDecimal("0.123"), JSONParser.parse("123E-3"));
+ }
+
+ @Test
+ public void testKeywords() throws JSONParser.JSONParseException {
+ assertNull(JSONParser.parse("null"));
+ assertEquals(true, JSONParser.parse("true"));
+ assertEquals(false, JSONParser.parse("false"));
+ try {
+ JSONParser.parse("NULL");
+ fail();
+ } catch (JSONParser.JSONParseException e) {
+ assertThat(e.getMessage(), containsString("quoted"));
+ }
+ }
+
+ @Test
+ public void testBlockComments() throws JSONParser.JSONParseException {
+ assertEquals(ImmutableList.of(1, 2), JSONParser.parse("/**/[/**/1/**/, /**/2/**/]/**/"));
+ assertEquals(ImmutableList.of(1, 2), JSONParser.parse("/*x*/[/*x*/1/*x*/, /*x*/2/*x*/]/*x*/"));
+ assertEquals(ImmutableList.of(1), JSONParser.parse(" /*x*/ /**//**/ [ /*x*/ /*\n*//***/ 1 ]"));
+ try {
+ JSONParser.parse("/*");
+ fail();
+ } catch (JSONParser.JSONParseException e) {
+ assertThat(e.getMessage(), containsString("Unclosed comment"));
+ }
+ try {
+ JSONParser.parse("[/*]");
+ fail();
+ } catch (JSONParser.JSONParseException e) {
+ assertThat(e.getMessage(), containsString("Unclosed comment"));
+ }
+ }
+
+ @Test
+ public void testLineComments() throws JSONParser.JSONParseException {
+ assertEquals(ImmutableList.of(1, 2), JSONParser.parse("//c1\n[ //c2\n1, //c3\n 2//c5\n] //c4"));
+ assertEquals(ImmutableList.of(1, 2), JSONParser.parse("// c1\n//\r// c2\r\n// c3\r\n[ 1, 2 ]//"));
+ assertEquals(ImmutableList.of(1, 2), JSONParser.parse("[1, 2]\n//\n"));
+ }
+
+ @Test
+ public void testWhitespace() throws JSONParser.JSONParseException {
+ assertEquals(ImmutableList.of(1, 2), JSONParser.parse(" [ 1 ,\n2 ] "));
+ assertEquals(ImmutableList.of(1, 2), JSONParser.parse("\uFEFF[\u00A01\u00A0,2]"));
+ }
+
+ @Test
+ public void testMixed() throws JSONParser.JSONParseException {
+ LinkedHashMap<String, Object> m = new LinkedHashMap<>();
+ m.put("x", 1);
+ m.put("y", null);
+ assertEquals(
+ ImmutableList.of(
+ ImmutableMap.of("a", Collections.emptyMap()),
+ ImmutableMap.of("b",
+ Arrays.asList(
+ m,
+ true,
+ null
+ ))
+ ),
+ JSONParser.parse("" +
+ "[\n" +
+ "{\"a\":{}},\n" +
+ "{\"b\":\n" +
+ "[" +
+ "{\"x\":1, \"y\": null}," +
+ "true," +
+ "null" +
+ "] // comment\n" +
+ "}\n" +
+ "]"));
+ }
+
+ private static void assertEquals(Object expected, TemplateModel actual) {
+ try {
+ Assert.assertEquals(expected, DeepUnwrap.unwrap(actual));
+ } catch (TemplateModelException e) {
+ throw new BugException(e);
+ }
+ }
+
+}
\ No newline at end of file