You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2026/07/16 20:51:18 UTC
(impala) 01/03: IMPALA-14912: Calcite Planner: Fix date string parsing
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit ecc7381315bb7098fca0b6d8a0fa0af8093f4b3b
Author: Steve Carlin <sc...@cloudera.com>
AuthorDate: Wed Apr 1 20:17:00 2026 -0700
IMPALA-14912: Calcite Planner: Fix date string parsing
This commit fixes parsing for the following date strings SQL
select DATE '1999-01-01';
select DATE "1999-01-01";
This essentially reverts an earlier fix, IMPALA-13525, which was
way more complicated than it needed to be. The Impala StringLiteral
already handles escaped strings, so there is no need to remove the
escapes within the Calcite parser. The changes in Parser.jj reflect
the major part of the reversion, as well as removing the ParserUtil.java
file.
A change also had to be made in the ImpalaRexExecutor where the constant
folding was done. This completes the UTF-8 constant folding commit for
the Calcite Planner (IMPALA-10349)
One more change was made with the unhex function. Unhex can return a
non-UTF8 string. Impala can handle this in a StringLiteral, but Calcite
cannot. Because of this, the constant folding for unhex when the string
is non-UTF8 is done at physical node creation time rather than during
the optimization phase.
Change-Id: Ia08071193f86423a0e548c6b6690993afdad6272
Reviewed-on: http://gerrit.cloudera.org:8080/24218
Reviewed-by: Aman Sinha <am...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../org/apache/impala/analysis/LiteralExpr.java | 12 ++
.../src/main/codegen/templates/Parser.jj | 19 +-
.../impala/calcite/functions/RexCallConverter.java | 25 +++
.../calcite/functions/RexLiteralConverter.java | 2 +-
.../impala/calcite/rel/node/ImpalaValuesRel.java | 2 +-
.../impala/calcite/rules/ImpalaRexExecutor.java | 23 +--
.../org/apache/impala/calcite/util/ParserUtil.java | 194 ---------------------
.../queries/QueryTest/calcite.test | 40 +++++
8 files changed, 95 insertions(+), 222 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/analysis/LiteralExpr.java b/fe/src/main/java/org/apache/impala/analysis/LiteralExpr.java
index 61ed141ae..4dee1ea8f 100644
--- a/fe/src/main/java/org/apache/impala/analysis/LiteralExpr.java
+++ b/fe/src/main/java/org/apache/impala/analysis/LiteralExpr.java
@@ -64,6 +64,16 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
*/
public static LiteralExpr createFromUnescapedStr(String value, Type type)
throws AnalysisException {
+ return createFromStr(value, type, false);
+ }
+
+ /**
+ * Creates an analyzed literal of 'type'. The boolean 'needsUnescaping' should be
+ * set to true if there are escape backslashes within the string.
+ * Returns null for types that do not have a LiteralExpr subclass, e.g. TIMESTAMP.
+ */
+ public static LiteralExpr createFromStr(String value, Type type,
+ boolean needsUnescaping) throws AnalysisException {
if (!type.isValid()) {
throw new UnsupportedFeatureException("Invalid literal type: " + type.toSql());
}
@@ -87,6 +97,8 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
case STRING:
case VARCHAR:
case CHAR:
+ e = new StringLiteral(value, type, needsUnescaping);
+ break;
case BINARY:
e = new StringLiteral(value, type, false);
break;
diff --git a/java/calcite-planner/src/main/codegen/templates/Parser.jj b/java/calcite-planner/src/main/codegen/templates/Parser.jj
index 4f064a051..418c1a2f6 100644
--- a/java/calcite-planner/src/main/codegen/templates/Parser.jj
+++ b/java/calcite-planner/src/main/codegen/templates/Parser.jj
@@ -126,7 +126,6 @@ import org.apache.impala.calcite.type.ImpalaSqlIntervalQualifier;
import org.apache.impala.calcite.operators.ImpalaCustomOperatorTable;
import org.apache.impala.calcite.operators.ImpalaConcatOrOperator;
import org.apache.impala.calcite.operators.ImpalaSqlNumericLiteral;
-import org.apache.impala.calcite.util.ParserUtil;
import org.apache.impala.catalog.ScalarType;
import com.google.common.collect.ImmutableList;
@@ -4623,33 +4622,25 @@ SqlNode StringLiteral() :
|
<BIG_QUERY_DOUBLE_QUOTED_STRING>
{
- p = SqlParserUtil.stripQuotes(getToken(0).image, DQ, DQ, "\\\"",
+ p = SqlParserUtil.stripQuotes(getToken(0).image, DQ, DQ, "\"",
Casing.UNCHANGED);
try {
- p = ParserUtil.replaceEscapedChars(p);
return SqlLiteral.createCharString(p, charSet, getPos());
} catch (java.nio.charset.UnsupportedCharsetException e) {
throw SqlUtil.newContextException(getPos(),
RESOURCE.unknownCharacterSet(charSet));
- } catch (ParserUtil.MalformedUnicodeEscape e) {
- throw SqlUtil.newContextException(getPos(),
- RESOURCE.unicodeEscapeMalformed(e.i));
}
}
|
<BIG_QUERY_QUOTED_STRING>
{
- p = SqlParserUtil.stripQuotes(getToken(0).image, "'", "'", "\\'",
+ p = SqlParserUtil.stripQuotes(getToken(0).image, "'", "'", "'",
Casing.UNCHANGED);
try {
- p = ParserUtil.replaceEscapedChars(p);
return SqlLiteral.createCharString(p, charSet, getPos());
} catch (java.nio.charset.UnsupportedCharsetException e) {
throw SqlUtil.newContextException(getPos(),
RESOURCE.unknownCharacterSet(charSet));
- } catch (ParserUtil.MalformedUnicodeEscape e) {
- throw SqlUtil.newContextException(getPos(),
- RESOURCE.unicodeEscapeMalformed(e.i));
}
}
}
@@ -4667,7 +4658,11 @@ String SimpleStringLiteral() :
}
|
<BIG_QUERY_QUOTED_STRING> {
- return SqlParserUtil.stripQuotes(token.image, "'", "'", "\\'", Casing.UNCHANGED);
+ return SqlParserUtil.stripQuotes(token.image, "'", "'", "'", Casing.UNCHANGED);
+ }
+|
+ <BIG_QUERY_DOUBLE_QUOTED_STRING> {
+ return SqlParserUtil.stripQuotes(token.image, "\"", "\"", "\"", Casing.UNCHANGED);
}
}
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexCallConverter.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexCallConverter.java
index 752c916c4..23c7a82e5 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexCallConverter.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexCallConverter.java
@@ -41,6 +41,7 @@ import org.apache.impala.analysis.IsNullPredicate;
import org.apache.impala.analysis.LikePredicate;
import org.apache.impala.analysis.LiteralExpr;
import org.apache.impala.analysis.NumericLiteral;
+import org.apache.impala.analysis.StringLiteral;
import org.apache.impala.analysis.TimestampArithmeticExpr;
import org.apache.impala.calcite.operators.ImpalaInOperator;
import org.apache.impala.calcite.rules.ImpalaRexExecutor;
@@ -54,6 +55,7 @@ import org.slf4j.LoggerFactory;
import java.math.BigDecimal;
import java.util.ArrayList;
+import java.util.HexFormat;
import java.util.List;
import java.util.Map;
@@ -131,6 +133,10 @@ public class RexCallConverter {
return createCompoundExpr(rexCall, params);
}
+ if (fn.functionName().equals("unhex")) {
+ return createUnhexExpr(fn, rexCall, params);
+ }
+
Type impalaRetType = ImpalaTypeConverter.createImpalaType(fn.getReturnType(),
rexCall.getType().getPrecision(), rexCall.getType().getScale());
@@ -196,6 +202,25 @@ public class RexCallConverter {
return new LikePredicate(likeOp, params.get(0), params.get(1));
}
+ private static Expr createUnhexExpr(Function fn, RexCall rexCall, List<Expr> params) {
+ // Constant folding fails for the unhex function when the bytes are non-UTF-8
+ // ("unhex('AA')" produces a non-UTF byte string. Constant folding for
+ // this function is done here.
+ if (params.get(0) instanceof StringLiteral) {
+ try {
+ HexFormat hex = HexFormat.of();
+ String s = ((StringLiteral)params.get(0)).getStringValue();
+ return new StringLiteral(hex.parseHex(s), Type.STRING);
+ } catch (Exception e) {
+ // If the string isn't a hex value, a blank string is created.
+ return new StringLiteral("", Type.STRING, false);
+ }
+ }
+ // When the parameter is not a literal, it will still be an operand
+ // (potentially a function) of type string.
+ return new AnalyzedFunctionCallExpr(fn, params, Type.STRING);
+ }
+
/**
* Create In Expr
*/
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexLiteralConverter.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexLiteralConverter.java
index f5501b550..8ff965a9e 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexLiteralConverter.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/RexLiteralConverter.java
@@ -99,7 +99,7 @@ public class RexLiteralConverter {
case CHAR:
case VARCHAR:
return new StringLiteral(rexLiteral.getValueAs(String.class),
- ImpalaTypeConverter.createImpalaType(rexLiteral.getType()), false);
+ ImpalaTypeConverter.createImpalaType(rexLiteral.getType()), true);
case DATE:
DateString dateStringClass = rexLiteral.getValueAs(DateString.class);
String dateString = (dateStringClass == null) ? null : dateStringClass.toString();
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaValuesRel.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaValuesRel.java
index 0d40bd30d..f9071cbb5 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaValuesRel.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaValuesRel.java
@@ -125,7 +125,7 @@ public class ImpalaValuesRel extends Values
return nullLiteral;
}
- return LiteralExpr.createFromUnescapedStr(expr.getStringValue(), impalaType);
+ return LiteralExpr.createFromStr(expr.getStringValue(), impalaType, true);
}
@Override
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaRexExecutor.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaRexExecutor.java
index 888a35963..8c4e51627 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaRexExecutor.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rules/ImpalaRexExecutor.java
@@ -39,9 +39,10 @@ import org.apache.impala.common.ImpalaException;
import org.apache.impala.service.FeSupport;
import org.apache.impala.thrift.TColumnValue;
import org.apache.impala.thrift.TQueryCtx;
+import org.apache.impala.util.StringUtils;
-import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
+import java.nio.ByteBuffer;
import java.util.List;
import org.apache.commons.lang3.exception.ExceptionUtils;
@@ -209,21 +210,15 @@ public class ImpalaRexExecutor implements RexExecutor {
byte[] bytes = new byte[colVal.binary_val.remaining()];
colVal.binary_val.get(bytes);
- // Converting strings between the BE/FE does not work properly for the
- // extended ASCII characters above 127. Bail in such cases to avoid
- // producing incorrect results.
- for (byte b: bytes) {
- if (b < 0) {
- return constExp;
- }
- }
- try {
- String newString = new String(bytes, "US-ASCII");
- return builder.makeLiteral(newString, returnType, true);
- } catch (UnsupportedEncodingException e) {
- LOG.debug("Could not interpret return value for " + colVal);
+ String newString = StringUtils.fromUtf8Buffer(ByteBuffer.wrap(bytes), true);
+ // In some cases, the bytes may not be Utf8 characters (e.g. unhex('AA'))
+ // If this happens, the expression cannot be reduced, so we just return
+ // the original expression.
+ if (newString == null) {
return constExp;
}
+ newString = newString.replace("\\", "\\\\");
+ return builder.makeLiteral(newString, returnType, true);
}
Preconditions.checkState(!colVal.isSetTimestamp_val(),
"Simplified into timestamp constant but this should not happen");
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/util/ParserUtil.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/util/ParserUtil.java
deleted file mode 100644
index bf034bd4d..000000000
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/util/ParserUtil.java
+++ /dev/null
@@ -1,194 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.impala.calcite.util;
-
-import org.apache.calcite.sql.parser.SqlParserUtil;
-
-import static java.lang.Integer.parseInt;
-import java.util.function.Predicate;
-
-/**
- * Util functions for parser
- *
- * IMPALA NOTE: This was copied from SqlParserUtil from Calcite 1.37 with one
- * modification. If there is a regexp character following the
- * backslash, we remove the backslash. The rest of the method is
- * copied as/is. The change is noted in a comment below (search for IMPALA)
- */
-public class ParserUtil {
- /**
- * replaceEscapeChars
- *
- * Converts the contents of a character literal with escapes like those used
- * in the C programming language to the corresponding Java string
- * representation.
- *
- * <p>If the literal "{@code E'a\tc'}" occurs in the SQL source text, then
- * this method will be invoked with the string "{@code a\tc}" (4 characters)
- * and will return a Java string with the three characters 'a', TAB, 'b'.
- *
- * @param input String that contains C-style escapes
- * @return String with escapes converted into Java characters
- * @throws MalformedUnicodeEscape if input contains invalid unicode escapes
- */
- public static String replaceEscapedChars(String input)
- throws MalformedUnicodeEscape {
- // The implementation of this method is based on Crate's method
- // Literals.replaceEscapedChars.
- final int length = input.length();
- if (length <= 1) {
- return input;
- }
- final StringBuilder builder = new StringBuilder(length);
- int endIdx;
- for (int i = 0; i < length; i++) {
- char currentChar = input.charAt(i);
- if (currentChar == '\\' && i + 1 < length) {
- char nextChar = input.charAt(i + 1);
- switch (nextChar) {
- case 'b':
- builder.append('\b');
- i++;
- break;
- case 'f':
- builder.append('\f');
- i++;
- break;
- case 'n':
- builder.append('\n');
- i++;
- break;
- case 'r':
- builder.append('\r');
- i++;
- break;
- case 't':
- builder.append('\t');
- i++;
- break;
- case '\\':
- case '\'':
- // IMPALA NOTE: Impala also allows escaping of these regexp characters. We
- // just remove the backslash here
- case '.':
- case '+':
- case '*':
- case '?':
- case '^':
- case '$':
- case '(':
- case ')':
- case '[':
- case ']':
- case '{':
- case '}':
- case '|':
- case '"':
- builder.append(nextChar);
- i++;
- break;
- case 'u':
- case 'U':
- // handle unicode case
- final int charsToConsume = (nextChar == 'u') ? 4 : 8;
- if (i + 1 + charsToConsume >= length) {
- throw new MalformedUnicodeEscape(i);
- }
- endIdx =
- calculateMaxCharsInSequence(input, i + 2, charsToConsume,
- SqlParserUtil::isHexDigit);
- if (endIdx != i + 2 + charsToConsume) {
- throw new MalformedUnicodeEscape(i);
- }
- builder.appendCodePoint(parseInt(input.substring(i + 2, endIdx), 16));
- i = endIdx - 1; // skip already consumed chars
- break;
- case 'x':
- // handle hex byte case - up to 2 chars for hex value
- endIdx =
- calculateMaxCharsInSequence(input, i + 2, 2,
- SqlParserUtil::isHexDigit);
- if (endIdx > i + 2) {
- builder.appendCodePoint(parseInt(input.substring(i + 2, endIdx), 16));
- i = endIdx - 1; // skip already consumed chars
- } else {
- // hex sequence unmatched - output original char
- builder.append(nextChar);
- i++;
- }
- break;
- case '0':
- case '1':
- case '2':
- case '3':
- // handle octal case - up to 3 chars
- endIdx =
- calculateMaxCharsInSequence(input, i + 2,
- 2, // first char is already "consumed"
- SqlParserUtil::isOctalDigit);
- builder.appendCodePoint(parseInt(input.substring(i + 1, endIdx), 8));
- i = endIdx - 1; // skip already consumed chars
- break;
- default:
- // non-valid escaped char sequence
- builder.append(currentChar);
- }
- } else {
- builder.append(currentChar);
- }
- }
- return builder.toString();
- }
-
- /**
- * Calculates the maximum number of consecutive characters of the
- * {@link CharSequence} argument, starting from {@code beginIndex}, that match
- * a given {@link Predicate}. The number of characters to match are either
- * capped from the {@code maxCharsToMatch} parameter or the sequence length.
- *
- * <p>Examples:
- * <pre>
- * {@code
- * calculateMaxCharsInSequence("12345", 0, 2, Character::isDigit) -> 2
- * calculateMaxCharsInSequence("12345", 3, 2, Character::isDigit) -> 5
- * calculateMaxCharsInSequence("12345", 4, 2, Character::isDigit) -> 5
- * }
- * </pre>
- *
- * @return the index of the first non-matching character
- */
- private static int calculateMaxCharsInSequence(CharSequence seq,
- int beginIndex,
- int maxCharsToMatch,
- Predicate<Character> predicate) {
- int idx = beginIndex;
- final int end = Math.min(seq.length(), beginIndex + maxCharsToMatch);
- while (idx < end && predicate.test(seq.charAt(idx))) {
- idx++;
- }
- return idx;
- }
- /** Thrown by {@link #replaceEscapedChars(String)}. */
- public static class MalformedUnicodeEscape extends Exception {
- public final int i;
-
- MalformedUnicodeEscape(int i) {
- this.i = i;
- }
- }
-}
diff --git a/testdata/workloads/functional-query/queries/QueryTest/calcite.test b/testdata/workloads/functional-query/queries/QueryTest/calcite.test
index dea209964..0b05a1cee 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/calcite.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/calcite.test
@@ -1337,3 +1337,43 @@ bigint
---- RUNTIME_PROFILE
row_regex: PlannerType: CalcitePlanner
====
+---- QUERY
+# IMPALA-14912: select date with quotes
+select DATE '1999-01-01';
+---- RESULTS
+1999-01-01
+---- TYPES
+date
+---- RUNTIME_PROFILE
+row_regex: PlannerType: CalcitePlanner
+====
+---- QUERY
+# IMPALA-14912: select date with double quotes
+select DATE "1999-01-01";
+---- RESULTS
+1999-01-01
+---- TYPES
+date
+---- RUNTIME_PROFILE
+row_regex: PlannerType: CalcitePlanner
+====
+---- QUERY
+select id, hex(cast(binary_col as string)) from functional.binary_tbl
+where binary_col = cast(unhex("FF4433221100") as binary)
+---- TYPES
+INT, STRING
+---- RESULTS
+8,'FF4433221100'
+---- RUNTIME_PROFILE
+row_regex: PlannerType: CalcitePlanner
+====
+---- QUERY
+# a non hex value produces a blank string.
+select unhex("FG");
+---- TYPES
+STRING
+---- RESULTS
+''
+---- RUNTIME_PROFILE
+row_regex: PlannerType: CalcitePlanner
+====