You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by sn...@apache.org on 2024/11/15 12:47:14 UTC
(flink) 02/02: [FLINK-31362][table] Upgrade Calcite to 1.33.0
This is an automated email from the ASF dual-hosted git repository.
snuyanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 77e7ac076c35d229fab89b8b6ae15138b0e41a22
Author: Sergey Nuyanzin <sn...@gmail.com>
AuthorDate: Wed Jan 31 17:35:50 2024 +0100
[FLINK-31362][table] Upgrade Calcite to 1.33.0
This closes #24255
---
.../docs/dev/table/functions/systemFunctions.md | 75 +-
.../docs/dev/table/sql/queries/overview.md | 15 +
.../docs/dev/table/functions/systemFunctions.md | 31 +-
.../content/docs/dev/table/sql/queries/overview.md | 15 +
docs/content/docs/dev/table/sql/queries/select.md | 2 +-
flink-table/flink-sql-parser/pom.xml | 9 +-
.../src/main/codegen/data/Parser.tdd | 1 +
.../src/main/codegen/templates/Parser.jj | 459 ++-
.../sql/parser/validate/FlinkSqlConformance.java | 10 +
.../flink/sql/parser/FlinkSqlParserImplTest.java | 119 +-
flink-table/flink-table-calcite-bridge/pom.xml | 16 +-
flink-table/flink-table-planner/pom.xml | 13 +-
.../calcite/rel/logical/LogicalSnapshot.java | 9 +-
.../calcite/rel/metadata/RelMdPredicates.java | 1 +
.../main/java/org/apache/calcite/rex/RexUtil.java | 3176 ++++++++++++++++++++
.../calcite/sql/fun/SqlTimestampAddFunction.java | 95 +-
.../type/CompositeSingleOperandTypeChecker.java | 117 +
.../calcite/sql/type/SqlTypeFactoryImpl.java | 11 +-
.../calcite/sql/validate/SqlValidatorImpl.java | 218 +-
.../apache/calcite/sql2rel/RelDecorrelator.java | 5 +-
.../apache/calcite/sql2rel/SqlToRelConverter.java | 197 +-
.../calcite/sql2rel/StandardConvertletTable.java | 174 +-
.../java/org/apache/calcite/tools/RelBuilder.java | 76 +-
.../src/main/resources/META-INF/NOTICE | 7 +-
.../planner/calcite/FlinkLogicalRelFactories.scala | 5 +-
.../plan/batch/sql/join/BroadcastHashJoinTest.xml | 19 +
.../plan/batch/sql/join/NestedLoopJoinTest.xml | 4 +-
.../plan/batch/sql/join/ShuffledHashJoinTest.xml | 19 +
.../plan/batch/sql/join/SortMergeJoinTest.xml | 19 +
.../JoinDependentConditionDerivationRuleTest.xml | 6 +-
.../logical/RemoveSingleAggregateRuleTest.xml | 2 +-
.../SubqueryCorrelateVariablesValidationTest.xml | 4 +-
.../planner/expressions/TemporalTypesTest.scala | 2 +-
.../batch/sql/join/BroadcastHashJoinTest.scala | 7 -
.../plan/batch/sql/join/ShuffledHashJoinTest.scala | 7 -
.../plan/batch/sql/join/SortMergeJoinTest.scala | 7 -
.../metadata/FlinkRelMdDistinctRowCountTest.scala | 12 +-
.../metadata/FlinkRelMdPopulationSizeTest.scala | 14 +-
.../JoinDependentConditionDerivationRuleTest.scala | 2 +-
.../table/validation/AggregateValidationTest.scala | 2 +-
.../planner/plan/utils/FlinkRelMdUtilTest.scala | 7 -
flink-table/pom.xml | 4 +-
42 files changed, 4493 insertions(+), 500 deletions(-)
diff --git a/docs/content.zh/docs/dev/table/functions/systemFunctions.md b/docs/content.zh/docs/dev/table/functions/systemFunctions.md
index f3721c49321..db989002717 100644
--- a/docs/content.zh/docs/dev/table/functions/systemFunctions.md
+++ b/docs/content.zh/docs/dev/table/functions/systemFunctions.md
@@ -118,43 +118,44 @@ JSON 函数使用符合 ISO/IEC TR 19075-6 SQL标准的 JSON 路径表达式。
下表列出了时间间隔单位和时间点单位标识符。
对于 Table API，请使用 `_` 代替空格（例如 `DAY_TO_HOUR`）。
-
-| 时间间隔单位 | 时间点单位 |
-| :------------------------ | :------------------------------ |
-| `MILLENNIUM` | |
-| `CENTURY` | |
-| `DECADE` | |
-| `YEAR` | `YEAR` |
-| `YEAR TO MONTH` | |
-| `QUARTER` | `QUARTER` |
-| `MONTH` | `MONTH` |
-| `WEEK` | `WEEK` |
-| `DAY` | `DAY` |
-| `DAY TO HOUR` | |
-| `DAY TO MINUTE` | |
-| `DAY TO SECOND` | |
-| `HOUR` | `HOUR` |
-| `HOUR TO MINUTE` | |
-| `HOUR TO SECOND` | |
-| `MINUTE` | `MINUTE` |
-| `MINUTE TO SECOND` | |
-| `SECOND` | `SECOND` |
-| `MILLISECOND` | `MILLISECOND` |
-| `MICROSECOND` | `MICROSECOND` |
-| `NANOSECOND` | |
-| `EPOCH` | |
-| `DOY` _（仅适用SQL）_ | |
-| `DOW` _（仅适用SQL）_ | |
-| `ISODOW` _（仅适用SQL）_ | |
-| `ISOYEAR` _（仅适用SQL）_ | |
-| | `SQL_TSI_YEAR` _（仅适用SQL）_ |
-| | `SQL_TSI_QUARTER` _（仅适用SQL）_ |
-| | `SQL_TSI_MONTH` _（仅适用SQL）_ |
-| | `SQL_TSI_WEEK` _（仅适用SQL）_ |
-| | `SQL_TSI_DAY` _（仅适用SQL）_ |
-| | `SQL_TSI_HOUR` _（仅适用SQL）_ |
-| | `SQL_TSI_MINUTE` _（仅适用SQL）_ |
-| | `SQL_TSI_SECOND ` _（仅适用SQL）_ |
+Plural works for SQL only.
+
+| 时间间隔单位 | 时间点单位 |
+|:-------------------------|:-----------------------------|
+| `MILLENNIUM` | |
+| `CENTURY` | |
+| `DECADE` | |
+| `YEAR(S)` | `YEAR` |
+| `YEAR(S) TO MONTH(S)` | |
+| `QUARTER(S)` | `QUARTER` |
+| `MONTH(S)` | `MONTH` |
+| `WEEK(S)` | `WEEK` |
+| `DAY(S)` | `DAY` |
+| `DAY(S) TO HOUR(S)` | |
+| `DAY(S) TO MINUTE(S)` | |
+| `DAY(S) TO SECOND(S)` | |
+| `HOUR(S)` | `HOUR` |
+| `HOUR(S) TO MINUTE(S)` | |
+| `HOUR(S) TO SECOND(S)` | |
+| `MINUTE(S)` | `MINUTE` |
+| `MINUTE(S) TO SECOND(S)` | |
+| `SECOND(S)` | `SECOND` |
+| `MILLISECOND` | `MILLISECOND` |
+| `MICROSECOND` | `MICROSECOND` |
+| `NANOSECOND` | |
+| `EPOCH` | |
+| `DOY` _（仅适用SQL）_ | |
+| `DOW` _（仅适用SQL）_ | |
+| `ISODOW` _（仅适用SQL）_ | |
+| `ISOYEAR` _（仅适用SQL）_ | |
+| | `SQL_TSI_YEAR` _（仅适用SQL）_ |
+| | `SQL_TSI_QUARTER` _（仅适用SQL）_ |
+| | `SQL_TSI_MONTH` _（仅适用SQL）_ |
+| | `SQL_TSI_WEEK` _（仅适用SQL）_ |
+| | `SQL_TSI_DAY` _（仅适用SQL）_ |
+| | `SQL_TSI_HOUR` _（仅适用SQL）_ |
+| | `SQL_TSI_MINUTE` _（仅适用SQL）_ |
+| | `SQL_TSI_SECOND ` _（仅适用SQL）_ |
{{< top >}}
diff --git a/docs/content.zh/docs/dev/table/sql/queries/overview.md b/docs/content.zh/docs/dev/table/sql/queries/overview.md
index 6f8b12193f0..d478ce446e5 100644
--- a/docs/content.zh/docs/dev/table/sql/queries/overview.md
+++ b/docs/content.zh/docs/dev/table/sql/queries/overview.md
@@ -426,6 +426,21 @@ Flink SQL> SELECT 'Hello World', 'It''s me';
- 使用反斜杠(`\`)作为转义字符 (默认)：`SELECT U&'\263A'`
- 使用自定义的转义字符：`SELECT U&'#263A' UESCAPE '#'`
+Starting Flink 2.0 there is C-style escape available
+
+| Backslash Escape Sequence | Interpretation |
+|:----------------------------------|:--------------------------------------------------|
+| \b | backspace |
+| \f | form feed |
+| \n | newline |
+| \r | carriage return |
+| \t | tab |
+| \o, \oo, \ooo (o = 0–7) | octal byte value |
+| \xh, \xhh (h = 0–9, A–F) | hexadecimal byte value |
+| \uxxxx, \Uxxxxxxxx (x = 0–9, A–F) | 16 or 32-bit hexadecimal Unicode character value |
+
+Example: `SELECT e'\u0061\x61\141' AS c` or `SELECT E'\u0061\x61\141' AS c`;
+
{{< top >}}
## 操作
diff --git a/docs/content/docs/dev/table/functions/systemFunctions.md b/docs/content/docs/dev/table/functions/systemFunctions.md
index 104aa47b9b2..7be1337f7b3 100644
--- a/docs/content/docs/dev/table/functions/systemFunctions.md
+++ b/docs/content/docs/dev/table/functions/systemFunctions.md
@@ -121,27 +121,28 @@ Time Interval and Point Unit Specifiers
The following table lists specifiers for time interval and time point units.
For Table API, please use `_` for spaces (e.g., `DAY_TO_HOUR`).
+Plural works for SQL only.
| Time Interval Unit | Time Point Unit |
|:-------------------------|:-------------------------------|
| `MILLENNIUM` | |
| `CENTURY` | |
| `DECADE` | |
-| `YEAR` | `YEAR` |
-| `YEAR TO MONTH` | |
-| `QUARTER` | `QUARTER` |
-| `MONTH` | `MONTH` |
-| `WEEK` | `WEEK` |
-| `DAY` | `DAY` |
-| `DAY TO HOUR` | |
-| `DAY TO MINUTE` | |
-| `DAY TO SECOND` | |
-| `HOUR` | `HOUR` |
-| `HOUR TO MINUTE` | |
-| `HOUR TO SECOND` | |
-| `MINUTE` | `MINUTE` |
-| `MINUTE TO SECOND` | |
-| `SECOND` | `SECOND` |
+| `YEAR(S)` | `YEAR` |
+| `YEAR(S) TO MONTH(S)` | |
+| `QUARTER(S)` | `QUARTER` |
+| `MONTH(S)` | `MONTH` |
+| `WEEK(S)` | `WEEK` |
+| `DAY(S)` | `DAY` |
+| `DAY(S) TO HOUR(S)` | |
+| `DAY(S) TO MINUTE(S)` | |
+| `DAY(S) TO SECOND(S)` | |
+| `HOUR(S)` | `HOUR` |
+| `HOUR(S) TO MINUTE(S)` | |
+| `HOUR(S) TO SECOND(S)` | |
+| `MINUTE(S)` | `MINUTE` |
+| `MINUTE(S) TO SECOND(S)` | |
+| `SECOND(S)` | `SECOND` |
| `MILLISECOND` | `MILLISECOND` |
| `MICROSECOND` | `MICROSECOND` |
| `NANOSECOND` | |
diff --git a/docs/content/docs/dev/table/sql/queries/overview.md b/docs/content/docs/dev/table/sql/queries/overview.md
index a5cebac833c..006db6eddd9 100644
--- a/docs/content/docs/dev/table/sql/queries/overview.md
+++ b/docs/content/docs/dev/table/sql/queries/overview.md
@@ -426,6 +426,21 @@ Unicode characters are supported in string literals. If explicit unicode code po
- Use the backslash (`\`) as escaping character (default): `SELECT U&'\263A'`
- Use a custom escaping character: `SELECT U&'#263A' UESCAPE '#'`
+Starting Flink 2.0 there is C-style escape available
+
+| Backslash Escape Sequence | Interpretation |
+|:----------------------------------|:--------------------------------------------------|
+| \b | backspace |
+| \f | form feed |
+| \n | newline |
+| \r | carriage return |
+| \t | tab |
+| \o, \oo, \ooo (o = 0–7) | octal byte value |
+| \xh, \xhh (h = 0–9, A–F) | hexadecimal byte value |
+| \uxxxx, \Uxxxxxxxx (x = 0–9, A–F) | 16 or 32-bit hexadecimal Unicode character value |
+
+Example: `SELECT e'\u0061\x61\141' AS c` or `SELECT E'\u0061\x61\141' AS c`;
+
{{< top >}}
## Operations
diff --git a/docs/content/docs/dev/table/sql/queries/select.md b/docs/content/docs/dev/table/sql/queries/select.md
index 4b29d6e4de6..33a40245060 100644
--- a/docs/content/docs/dev/table/sql/queries/select.md
+++ b/docs/content/docs/dev/table/sql/queries/select.md
@@ -32,7 +32,7 @@ The general syntax of the `SELECT` statement is:
SELECT select_list FROM table_expression [ WHERE boolean_expression ]
```
-The `table_expression` refers to any source of data. It could be an existing table, view, or `VALUES` clause, the joined results of multiple existing tables, or a subquery. Assuming that the table is available in the catalog, the following would read all rows from `Orders`.
+The `table_expression` refers to any source of data. It could be an existing table, view, `VALUES`, or `VALUE` clause, the joined results of multiple existing tables, or a subquery. Assuming that the table is available in the catalog, the following would read all rows from `Orders`.
```sql
SELECT * FROM Orders
diff --git a/flink-table/flink-sql-parser/pom.xml b/flink-table/flink-sql-parser/pom.xml
index f7f10cc3dcb..4e7c463db78 100644
--- a/flink-table/flink-sql-parser/pom.xml
+++ b/flink-table/flink-sql-parser/pom.xml
@@ -62,12 +62,13 @@ under the License.
<version>${calcite.version}</version>
<exclusions>
<!--
- "mvn dependency:tree" as of Calcite 1.32.0:
+ "mvn dependency:tree" as of Calcite 1.33.0:
- [INFO] +- org.apache.calcite:calcite-core:jar:1.32.0:compile
- [INFO] | +- org.apache.calcite.avatica:avatica-core:jar:1.22.0:compile
+ [INFO] +- org.apache.calcite:calcite-core:jar:1.33.0:compile
+ [INFO] | +- org.apache.calcite.avatica:avatica-core:jar:1.23.0:compile
[INFO] | +- org.apiguardian:apiguardian-api:jar:1.1.2:compile
- [INFO] | \- org.checkerframework:checker-qual:jar:3.12.0:compile
+ [INFO] | +- org.checkerframework:checker-qual:jar:3.12.0:compile
+ [INFO] | \- org.apache.commons:commons-math3:jar:3.6.1:runtime
Dependencies that are not needed for how we use Calcite right now.
-->
diff --git a/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd b/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd
index 64c450d573e..2a366116e93 100644
--- a/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd
+++ b/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd
@@ -708,4 +708,5 @@
includeCompoundIdentifier: true
includeBraces: true
includeAdditionalDeclarations: false
+ includeParsingStringLiteralAsArrayLiteral: false
}
diff --git a/flink-table/flink-sql-parser/src/main/codegen/templates/Parser.jj b/flink-table/flink-sql-parser/src/main/codegen/templates/Parser.jj
index b299a703457..57c5eec2373 100644
--- a/flink-table/flink-sql-parser/src/main/codegen/templates/Parser.jj
+++ b/flink-table/flink-sql-parser/src/main/codegen/templates/Parser.jj
@@ -161,7 +161,6 @@ public class ${parser.class} extends SqlAbstractParserImpl
private Casing unquotedCasing;
private Casing quotedCasing;
private int identifierMaxLength;
- private ImmutableMap<String, TimeUnit> timeUnitCodes;
private SqlConformance conformance;
/**
@@ -223,10 +222,6 @@ public class ${parser.class} extends SqlAbstractParserImpl
this.identifierMaxLength = identifierMaxLength;
}
- public void setTimeUnitCodes(Map<String, TimeUnit> timeUnitCodes) {
- this.timeUnitCodes = ImmutableMap.copyOf(timeUnitCodes);
- }
-
public void setConformance(SqlConformance conformance) {
this.conformance = conformance;
}
@@ -243,6 +238,17 @@ public class ${parser.class} extends SqlAbstractParserImpl
return SqlStmtList();
}
+ public SqlNode parseArray() throws SqlParseException {
+ switchTo(LexicalState.BQID);
+ try {
+ return ArrayLiteral();
+ } catch (ParseException ex) {
+ throw normalizeException(ex);
+ } catch (TokenMgrError ex) {
+ throw normalizeException(ex);
+ }
+ }
+
private SqlNode extend(SqlNode table, SqlNodeList extendList) {
return SqlStdOperatorTable.EXTEND.createCall(
Span.of(table, extendList).pos(), table, extendList);
@@ -2437,7 +2443,17 @@ SqlNode TableConstructor() :
final Span s;
}
{
- <VALUES> { s = span(); }
+ (
+ <VALUES> { s = span(); }
+ |
+ <VALUE>
+ {
+ s = span();
+ if (!this.conformance.isValueAllowed()) {
+ throw SqlUtil.newContextException(getPos(), RESOURCE.valueNotAllowed());
+ }
+ }
+ )
AddRowConstructor(list)
(
LOOKAHEAD(2)
@@ -3485,9 +3501,9 @@ SqlNode LeafQueryOrExpr(ExprContext exprContext) :
SqlNode e;
}
{
- e = Expression(exprContext) { return e; }
-|
e = LeafQuery(exprContext) { return e; }
+|
+ e = Expression(exprContext) { return e; }
}
/** As {@link #Expression} but appends to a list. */
@@ -4503,6 +4519,17 @@ SqlNode StringLiteral() :
return SqlStdOperatorTable.LITERAL_CHAIN.createCall(pos2, rands);
}
}
+|
+ <C_STYLE_ESCAPED_STRING_LITERAL>
+ {
+ try {
+ p = SqlParserUtil.parseCString(getToken(0).image);
+ } catch (SqlParserUtil.MalformedUnicodeEscape e) {
+ throw SqlUtil.newContextException(getPos(),
+ RESOURCE.unicodeEscapeMalformed(e.i));
+ }
+ return SqlLiteral.createCharString(p, "UTF16", getPos());
+ }
|
<BIG_QUERY_DOUBLE_QUOTED_STRING>
{
@@ -4581,15 +4608,24 @@ SqlLiteral DateTimeLiteral() :
}
|
<DATE> { s = span(); } p = SimpleStringLiteral() {
- return SqlParserUtil.parseDateLiteral(p, s.end(this));
+ return SqlLiteral.createUnknown("DATE", p, s.end(this));
+ }
+|
+ <DATETIME> { s = span(); } p = SimpleStringLiteral() {
+ return SqlLiteral.createUnknown("DATETIME", p, s.end(this));
}
|
<TIME> { s = span(); } p = SimpleStringLiteral() {
- return SqlParserUtil.parseTimeLiteral(p, s.end(this));
+ return SqlLiteral.createUnknown("TIME", p, s.end(this));
}
|
+ LOOKAHEAD(2)
<TIMESTAMP> { s = span(); } p = SimpleStringLiteral() {
- return SqlParserUtil.parseTimestampLiteral(p, s.end(this));
+ return SqlLiteral.createUnknown("TIMESTAMP", p, s.end(this));
+ }
+|
+ <TIMESTAMP> { s = span(); } <WITH> <LOCAL> <TIME> <ZONE> p = SimpleStringLiteral() {
+ return SqlLiteral.createUnknown("TIMESTAMP WITH LOCAL TIME ZONE", p, s.end(this));
}
}
@@ -4630,6 +4666,7 @@ SqlNode ArrayConstructor() :
SqlNodeList args;
SqlNode e;
final Span s;
+ final String p;
}
{
<ARRAY> { s = span(); }
@@ -4656,9 +4693,42 @@ SqlNode ArrayConstructor() :
return SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR.createCall(
s.end(this), args.getList());
}
+<#if (parser.includeParsingStringLiteralAsArrayLiteral!default.parser.includeParsingStringLiteralAsArrayLiteral) >
+ |
+ p = SimpleStringLiteral() {
+ try {
+ return SqlParserUtil.parseArrayLiteral(p);
+ } catch (SqlParseException ex) {
+ throw SqlUtil.newContextException(getPos(),
+ RESOURCE.illegalArrayExpression(p));
+ }
+ }
+</#if>
)
}
+SqlCall ArrayLiteral() :
+{
+ final List<SqlNode> list;
+ SqlNode e;
+ final Span s;
+}
+{
+ <LBRACE> { s = span(); }
+ (
+ e = Literal() { list = startList(e); }
+ ( <COMMA> e = Literal() { list.add(e); } )*
+ |
+ e = ArrayLiteral() { list = startList(e); }
+ ( <COMMA> e = ArrayLiteral() { list.add(e); } )*
+ |
+ { list = Collections.emptyList(); }
+ )
+ <RBRACE> {
+ return SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR.createCall(s.end(this), list);
+ }
+}
+
/** Parses a MAP constructor */
SqlNode MapConstructor() :
{
@@ -4791,6 +4861,15 @@ TimeUnit Year() :
<YEARS> { return warn(TimeUnit.YEAR); }
}
+TimeUnit Quarter() :
+{
+}
+{
+ <QUARTER> { return TimeUnit.QUARTER; }
+|
+ <QUARTERS> { return warn(TimeUnit.QUARTER); }
+}
+
TimeUnit Month() :
{
}
@@ -4800,6 +4879,15 @@ TimeUnit Month() :
<MONTHS> { return warn(TimeUnit.MONTH); }
}
+TimeUnit Week() :
+{
+}
+{
+ <WEEK> { return TimeUnit.WEEK; }
+|
+ <WEEKS> { return warn(TimeUnit.WEEK); }
+}
+
TimeUnit Day() :
{
}
@@ -4851,9 +4939,15 @@ SqlIntervalQualifier IntervalQualifier() :
LOOKAHEAD(2) <TO> end = Month()
| { end = null; }
)
+ |
+ start = Quarter() { s = span(); } startPrec = PrecisionOpt()
+ { end = null; }
|
start = Month() { s = span(); } startPrec = PrecisionOpt()
{ end = null; }
+ |
+ start = Week() { s = span(); } startPrec = PrecisionOpt()
+ { end = null; }
|
start = Day() { s = span(); } startPrec = PrecisionOpt()
(
@@ -4915,7 +5009,9 @@ SqlIntervalQualifier IntervalQualifierStart() :
(
(
start = Year()
+ | start = Quarter()
| start = Month()
+ | start = Week()
| start = Day()
| start = Hour()
| start = Minute()
@@ -4935,103 +5031,75 @@ SqlIntervalQualifier IntervalQualifierStart() :
}
}
-/**
- * Parses time unit for CEIL and FLOOR functions.
- */
-TimeUnit TimeUnit() :
-{
- final TimeUnit unit;
-}
-{
- LOOKAHEAD(1)
- (
- <MILLISECOND> { return TimeUnit.MILLISECOND; }
- | <SECOND> { return TimeUnit.SECOND; }
- | <MINUTE> { return TimeUnit.MINUTE; }
- | <HOUR> { return TimeUnit.HOUR; }
- | <DAY> { return TimeUnit.DAY; }
- | <DOW> { return TimeUnit.DOW; }
- | <DOY> { return TimeUnit.DOY; }
- | <ISODOW> { return TimeUnit.ISODOW; }
- | <ISOYEAR> { return TimeUnit.ISOYEAR; }
- | <WEEK> { return TimeUnit.WEEK; }
- | <MONTH> { return TimeUnit.MONTH; }
- | <QUARTER> { return TimeUnit.QUARTER; }
- | <YEAR> { return TimeUnit.YEAR; }
- | <EPOCH> { return TimeUnit.EPOCH; }
- | <DECADE> { return TimeUnit.DECADE; }
- | <CENTURY> { return TimeUnit.CENTURY; }
- | <MILLENNIUM> { return TimeUnit.MILLENNIUM; }
- )
-|
- unit = TimeUnitIdentifier() { return unit; }
-}
-
-/**
- * Parses time unit for the EXTRACT function.
- * As for FLOOR and CEIL, but also includes NANOSECOND and MICROSECOND.
+/** Parses a built-in time unit (e.g. "YEAR")
+ * or user-defined time frame (e.g. "MINUTE15")
+ * and in each case returns a {@link SqlIntervalQualifier}.
+ *
+ * <p>The units are used in several functions, incuding CEIL, FLOOR, EXTRACT.
+ * Includes NANOSECOND, MILLISECOND, which were previously allowed in EXTRACT
+ * but not CEIL, FLOOR.
+ *
+ * <p>Includes {@code WEEK} and {@code WEEK(SUNDAY)} through
+ {@code WEEK(SATURDAY)}.
+ *
+ * <p>Does not include SQL_TSI_DAY, SQL_TSI_FRAC_SECOND etc. These will be
+ * parsed as identifiers and can be resolved in the validator if they are
+ * registered as abbreviations in your time frame set.
*/
-TimeUnit TimeUnitForExtract() :
-{
+SqlIntervalQualifier TimeUnitOrName() : {
+ final Span span;
+ final String w;
final TimeUnit unit;
+ final SqlIdentifier unitName;
}
{
- LOOKAHEAD(1)
+ LOOKAHEAD(2)
+ <NANOSECOND> { return new SqlIntervalQualifier(TimeUnit.NANOSECOND, null, getPos()); }
+| <MICROSECOND> { return new SqlIntervalQualifier(TimeUnit.MICROSECOND, null, getPos()); }
+| <MILLISECOND> { return new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, getPos()); }
+| <SECOND> { return new SqlIntervalQualifier(TimeUnit.SECOND, null, getPos()); }
+| <MINUTE> { return new SqlIntervalQualifier(TimeUnit.MINUTE, null, getPos()); }
+| <HOUR> { return new SqlIntervalQualifier(TimeUnit.HOUR, null, getPos()); }
+| <DAY> { return new SqlIntervalQualifier(TimeUnit.DAY, null, getPos()); }
+| <DOW> { return new SqlIntervalQualifier(TimeUnit.DOW, null, getPos()); }
+| <DOY> { return new SqlIntervalQualifier(TimeUnit.DOY, null, getPos()); }
+| <ISODOW> { return new SqlIntervalQualifier(TimeUnit.ISODOW, null, getPos()); }
+| <ISOYEAR> { return new SqlIntervalQualifier(TimeUnit.ISOYEAR, null, getPos()); }
+| <WEEK> { span = span(); }
(
- <NANOSECOND> { return TimeUnit.NANOSECOND; }
- | <MICROSECOND> { return TimeUnit.MICROSECOND; }
+ LOOKAHEAD(2)
+ <LPAREN> w = weekdayName() <RPAREN> {
+ return new SqlIntervalQualifier(w, span.end(this));
+ }
+ |
+ { return new SqlIntervalQualifier(TimeUnit.WEEK, null, getPos()); }
)
-|
- unit = TimeUnit() { return unit; }
+| <MONTH> { return new SqlIntervalQualifier(TimeUnit.MONTH, null, getPos()); }
+| <QUARTER> { return new SqlIntervalQualifier(TimeUnit.QUARTER, null, getPos()); }
+| <YEAR> { return new SqlIntervalQualifier(TimeUnit.YEAR, null, getPos()); }
+| <EPOCH> { return new SqlIntervalQualifier(TimeUnit.EPOCH, null, getPos()); }
+| <DECADE> { return new SqlIntervalQualifier(TimeUnit.DECADE, null, getPos()); }
+| <CENTURY> { return new SqlIntervalQualifier(TimeUnit.CENTURY, null, getPos()); }
+| <MILLENNIUM> { return new SqlIntervalQualifier(TimeUnit.MILLENNIUM, null, getPos()); }
+| unitName = SimpleIdentifier() {
+ return new SqlIntervalQualifier(unitName.getSimple(),
+ unitName.getParserPosition());
+ }
}
-/**
- * Parses a simple identifier as a TimeUnit.
- */
-TimeUnit TimeUnitIdentifier() :
+String weekdayName() :
{
- final List<String> names = new ArrayList<String>();
- final List<SqlParserPos> positions = new ArrayList<SqlParserPos>();
}
{
- AddIdentifierSegment(names, positions) {
- TimeUnit unit = timeUnitCodes.get(names.get(0));
- if (unit != null) {
- return unit;
- }
- throw SqlUtil.newContextException(positions.get(0),
- RESOURCE.invalidDatetimeFormat(SqlIdentifier.getString(names)));
- }
+ <SUNDAY> { return "WEEK_SUNDAY"; }
+| <MONDAY> { return "WEEK_MONDAY"; }
+| <TUESDAY> { return "WEEK_TUESDAY"; }
+| <WEDNESDAY> { return "WEEK_WEDNESDAY"; }
+| <THURSDAY> { return "WEEK_THURSDAY"; }
+| <FRIDAY> { return "WEEK_FRIDAY"; }
+| <SATURDAY> { return "WEEK_SATURDAY"; }
}
-TimeUnit TimestampInterval() :
-{}
-{
- <FRAC_SECOND> { return TimeUnit.MICROSECOND; }
-| <MICROSECOND> { return TimeUnit.MICROSECOND; }
-| <NANOSECOND> { return TimeUnit.NANOSECOND; }
-| <SQL_TSI_FRAC_SECOND> { return TimeUnit.NANOSECOND; }
-| <SQL_TSI_MICROSECOND> { return TimeUnit.MICROSECOND; }
-| <SECOND> { return TimeUnit.SECOND; }
-| <SQL_TSI_SECOND> { return TimeUnit.SECOND; }
-| <MINUTE> { return TimeUnit.MINUTE; }
-| <SQL_TSI_MINUTE> { return TimeUnit.MINUTE; }
-| <HOUR> { return TimeUnit.HOUR; }
-| <SQL_TSI_HOUR> { return TimeUnit.HOUR; }
-| <DAY> { return TimeUnit.DAY; }
-| <SQL_TSI_DAY> { return TimeUnit.DAY; }
-| <WEEK> { return TimeUnit.WEEK; }
-| <SQL_TSI_WEEK> { return TimeUnit.WEEK; }
-| <MONTH> { return TimeUnit.MONTH; }
-| <SQL_TSI_MONTH> { return TimeUnit.MONTH; }
-| <QUARTER> { return TimeUnit.QUARTER; }
-| <SQL_TSI_QUARTER> { return TimeUnit.QUARTER; }
-| <YEAR> { return TimeUnit.YEAR; }
-| <SQL_TSI_YEAR> { return TimeUnit.YEAR; }
-}
-
-
-
/**
* Parses a dynamic parameter marker.
*/
@@ -5860,7 +5928,7 @@ SqlNode BuiltinFunctionCall() :
SqlNode e;
final Span s;
SqlDataTypeSpec dt;
- final TimeUnit unit;
+ final SqlIntervalQualifier unit;
final SqlNode node;
}
{
@@ -5879,8 +5947,8 @@ SqlNode BuiltinFunctionCall() :
}
|
<EXTRACT> { s = span(); }
- <LPAREN> unit = TimeUnitForExtract() {
- args.add(new SqlIntervalQualifier(unit, null, getPos()));
+ <LPAREN> unit = TimeUnitOrName() {
+ args.add(unit);
}
<FROM>
AddExpression(args, ExprContext.ACCEPT_SUB_QUERY)
@@ -6017,11 +6085,21 @@ SqlNode BuiltinFunctionCall() :
<RPAREN> {
return SqlStdOperatorTable.TRIM.createCall(s.end(this), args);
}
+ |
+ node = DateTruncFunctionCall() { return node; }
|
node = TimestampAddFunctionCall() { return node; }
|
node = TimestampDiffFunctionCall() { return node; }
|
+ node = TimestampDiff3FunctionCall() { return node; }
+ |
+ node = TimestampTruncFunctionCall() { return node; }
+ |
+ node = TimeDiffFunctionCall() { return node; }
+ |
+ node = TimeTruncFunctionCall() { return node; }
+ |
<#list (parser.builtinFunctionCallMethods!default.parser.builtinFunctionCallMethods) as method>
node = ${method} { return node; }
|
@@ -6291,7 +6369,6 @@ SqlNode JsonQueryWrapperBehavior() :
SqlCall JsonQueryFunctionCall() :
{
final SqlNode[] args = new SqlNode[6];
- SqlNode returningClause;
SqlNode e;
List<SqlNode> commonSyntax;
final Span span;
@@ -6533,14 +6610,12 @@ SqlCall TimestampAddFunctionCall() :
{
final List<SqlNode> args = new ArrayList<SqlNode>();
final Span s;
- final TimeUnit interval;
+ final SqlIntervalQualifier unit;
}
{
<TIMESTAMPADD> { s = span(); }
<LPAREN>
- interval = TimestampInterval() {
- args.add(SqlLiteral.createSymbol(interval, getPos()));
- }
+ unit = TimeUnitOrName() { args.add(unit); }
<COMMA>
AddExpression(args, ExprContext.ACCEPT_SUB_QUERY)
<COMMA>
@@ -6558,14 +6633,12 @@ SqlCall TimestampDiffFunctionCall() :
{
final List<SqlNode> args = new ArrayList<SqlNode>();
final Span s;
- final TimeUnit interval;
+ final SqlIntervalQualifier unit;
}
{
<TIMESTAMPDIFF> { s = span(); }
<LPAREN>
- interval = TimestampInterval() {
- args.add(SqlLiteral.createSymbol(interval, getPos()));
- }
+ unit = TimeUnitOrName() { args.add(unit); }
<COMMA>
AddExpression(args, ExprContext.ACCEPT_SUB_QUERY)
<COMMA>
@@ -6576,11 +6649,127 @@ SqlCall TimestampDiffFunctionCall() :
}
}
+/**
+ * Parses a call to BigQuery's TIMESTAMP_DIFF.
+ *
+ * <p>The difference between TIMESTAMPDIFF and TIMESTAMP_DIFF is the ordering of
+ * the parameters and the arrangement of the subtraction.
+ * TIMESTAMPDIFF uses (unit, timestamp1, timestamp2) with (t2 - t1), while
+ * TIMESTAMP_DIFF uses (timestamp1, timestamp2, unit) with (t1 - t2).
+ */
+SqlCall TimestampDiff3FunctionCall() :
+{
+ final List<SqlNode> args = new ArrayList<SqlNode>();
+ final Span s;
+ final SqlIntervalQualifier unit;
+}
+{
+ <TIMESTAMP_DIFF> { s = span(); }
+ <LPAREN>
+ AddExpression(args, ExprContext.ACCEPT_SUB_QUERY)
+ <COMMA>
+ AddExpression(args, ExprContext.ACCEPT_SUB_QUERY)
+ <COMMA>
+ unit = TimeUnitOrName() { args.add(unit); }
+ <RPAREN> {
+ return SqlLibraryOperators.TIMESTAMP_DIFF3.createCall(s.end(this), args);
+ }
+}
+
+/**
+ * Parses a call to DATE_TRUNC.
+ */
+SqlCall DateTruncFunctionCall() :
+{
+ final List<SqlNode> args = new ArrayList<SqlNode>();
+ final Span s;
+ final SqlIntervalQualifier unit;
+}
+{
+ <DATE_TRUNC> { s = span(); }
+ <LPAREN>
+ AddExpression(args, ExprContext.ACCEPT_SUB_QUERY)
+ <COMMA>
+ // A choice of arguments allows us to support both
+ // the BigQuery variant, e.g. "DATE_TRUNC(d, YEAR)",
+ // and the Redshift variant, e.g. "DATE_TRUNC('year', DATE '2008-09-08')".
+ (
+ unit = TimeUnitOrName() { args.add(unit); }
+ |
+ AddExpression(args, ExprContext.ACCEPT_SUB_QUERY)
+ )
+ <RPAREN> {
+ return SqlLibraryOperators.DATE_TRUNC.createCall(s.end(this), args);
+ }
+}
+
+/**
+ * Parses a call to TIMESTAMP_TRUNC.
+ */
+SqlCall TimestampTruncFunctionCall() :
+{
+ final List<SqlNode> args = new ArrayList<SqlNode>();
+ final Span s;
+ final SqlIntervalQualifier unit;
+}
+{
+ <TIMESTAMP_TRUNC> { s = span(); }
+ <LPAREN>
+ AddExpression(args, ExprContext.ACCEPT_SUB_QUERY)
+ <COMMA>
+ unit = TimeUnitOrName() { args.add(unit); }
+ <RPAREN> {
+ return SqlLibraryOperators.TIMESTAMP_TRUNC.createCall(s.end(this), args);
+ }
+}
+
+/**
+ * Parses a call to BigQuery's TIME_DIFF.
+ */
+SqlCall TimeDiffFunctionCall() :
+{
+ final List<SqlNode> args = new ArrayList<SqlNode>();
+ final Span s;
+ final SqlIntervalQualifier unit;
+}
+{
+ <TIME_DIFF> { s = span(); }
+ <LPAREN>
+ AddExpression(args, ExprContext.ACCEPT_SUB_QUERY)
+ <COMMA>
+ AddExpression(args, ExprContext.ACCEPT_SUB_QUERY)
+ <COMMA>
+ unit = TimeUnitOrName() { args.add(unit); }
+ <RPAREN> {
+ return SqlLibraryOperators.TIME_DIFF.createCall(s.end(this), args);
+ }
+}
+
+/**
+ * Parses a call to TIME_TRUNC.
+ */
+SqlCall TimeTruncFunctionCall() :
+{
+ final List<SqlNode> args = new ArrayList<SqlNode>();
+ final Span s;
+ final SqlIntervalQualifier unit;
+}
+{
+ <TIME_TRUNC> { s = span(); }
+ <LPAREN>
+ AddExpression(args, ExprContext.ACCEPT_SUB_QUERY)
+ <COMMA>
+ unit = TimeUnitOrName() { args.add(unit); }
+ <RPAREN> {
+ return SqlLibraryOperators.TIME_TRUNC.createCall(s.end(this), args);
+ }
+}
+
/**
* Parses a call to a grouping function inside the GROUP BY clause,
* for example {@code TUMBLE(rowtime, INTERVAL '1' MINUTE)}.
*/
-SqlCall GroupByWindowingCall():
+SqlCall GroupByWindowingCall() :
{
final Span s;
final List<SqlNode> args;
@@ -6875,16 +7064,15 @@ SqlNode StandardFloorCeilOptions(Span s, boolean floorFlag) :
{
SqlNode e;
final List<SqlNode> args = new ArrayList<SqlNode>();
- TimeUnit unit;
+ final SqlIntervalQualifier unit;
SqlCall function;
final Span s1;
}
{
<LPAREN> AddExpression(args, ExprContext.ACCEPT_SUB_QUERY)
(
- <TO>
- unit = TimeUnit() {
- args.add(new SqlIntervalQualifier(unit, null, getPos()));
+ <TO> unit = TimeUnitOrName() {
+ args.add(unit);
}
)?
<RPAREN> {
@@ -7069,6 +7257,36 @@ SqlNode JdbcFunctionCall() :
name = call.getOperator().getName();
args = new SqlNodeList(call.getOperandList(), getPos());
}
+ |
+ LOOKAHEAD(1)
+ call = DateTruncFunctionCall() {
+ name = call.getOperator().getName();
+ args = new SqlNodeList(call.getOperandList(), getPos());
+ }
+ |
+ LOOKAHEAD(1)
+ call = TimestampTruncFunctionCall() {
+ name = call.getOperator().getName();
+ args = new SqlNodeList(call.getOperandList(), getPos());
+ }
+ |
+ LOOKAHEAD(1)
+ call = TimeTruncFunctionCall() {
+ name = call.getOperator().getName();
+ args = new SqlNodeList(call.getOperandList(), getPos());
+ }
+ |
+ LOOKAHEAD(1)
+ call = TimestampDiff3FunctionCall() {
+ name = call.getOperator().getName();
+ args = new SqlNodeList(call.getOperandList(), getPos());
+ }
+ |
+ LOOKAHEAD(1)
+ call = TimeDiffFunctionCall() {
+ name = call.getOperator().getName();
+ args = new SqlNodeList(call.getOperandList(), getPos());
+ }
|
LOOKAHEAD(3)
call = TimestampDiffFunctionCall() {
@@ -7438,6 +7656,8 @@ SqlPostfixOperator PostfixRowOperator() :
| < DATA: "DATA" >
| < DATABASE: "DATABASE" >
| < DATE: "DATE" >
+| < DATE_TRUNC: "DATE_TRUNC" >
+| < DATETIME: "DATETIME" >
| < DATETIME_INTERVAL_CODE: "DATETIME_INTERVAL_CODE" >
| < DATETIME_INTERVAL_PRECISION: "DATETIME_INTERVAL_PRECISION" >
| < DAY: "DAY" >
@@ -7522,6 +7742,7 @@ SqlPostfixOperator PostfixRowOperator() :
| < FRAC_SECOND: "FRAC_SECOND" >
| < FRAME_ROW: "FRAME_ROW" >
| < FREE: "FREE" >
+| < FRIDAY: "FRIDAY" >
| < FROM: "FROM" > { beforeTableName(); }
| < FULL: "FULL" >
| < FUNCTION: "FUNCTION" >
@@ -7640,6 +7861,7 @@ SqlPostfixOperator PostfixRowOperator() :
| < MOD: "MOD" >
| < MODIFIES: "MODIFIES" >
| < MODULE: "MODULE" >
+| < MONDAY: "MONDAY" >
| < MONTH: "MONTH" >
| < MONTHS: "MONTHS" >
| < MORE_: "MORE" >
@@ -7737,6 +7959,7 @@ SqlPostfixOperator PostfixRowOperator() :
| < PROCEDURE: "PROCEDURE" >
| < PUBLIC: "PUBLIC" >
| < QUARTER: "QUARTER" >
+| < QUARTERS: "QUARTERS" >
| < RANGE: "RANGE" >
| < RANK: "RANK" >
| < READ: "READ" >
@@ -7786,6 +8009,7 @@ SqlPostfixOperator PostfixRowOperator() :
| < ROW_NUMBER: "ROW_NUMBER" >
| < ROWS: "ROWS" >
| < RUNNING: "RUNNING" >
+| < SATURDAY: "SATURDAY" >
| < SAVEPOINT: "SAVEPOINT" >
| < SCALAR: "SCALAR" >
| < SCALE: "SCALE" >
@@ -7899,6 +8123,7 @@ SqlPostfixOperator PostfixRowOperator() :
| < SUBSTRING_REGEX: "SUBSTRING_REGEX" >
| < SUCCEEDS: "SUCCEEDS" >
| < SUM: "SUM" >
+| < SUNDAY: "SUNDAY" >
| < SYMMETRIC: "SYMMETRIC" >
| < SYSTEM: "SYSTEM" >
| < SYSTEM_TIME: "SYSTEM_TIME" >
@@ -7908,11 +8133,16 @@ SqlPostfixOperator PostfixRowOperator() :
| < TABLESAMPLE: "TABLESAMPLE" >
| < TEMPORARY: "TEMPORARY" >
| < THEN: "THEN" >
+| < THURSDAY: "THURSDAY" >
| < TIES: "TIES" >
| < TIME: "TIME" >
+| < TIME_DIFF: "TIME_DIFF" >
+| < TIME_TRUNC: "TIME_TRUNC" >
| < TIMESTAMP: "TIMESTAMP" >
| < TIMESTAMPADD: "TIMESTAMPADD" >
| < TIMESTAMPDIFF: "TIMESTAMPDIFF" >
+| < TIMESTAMP_DIFF: "TIMESTAMP_DIFF" >
+| < TIMESTAMP_TRUNC: "TIMESTAMP_TRUNC" >
| < TIMEZONE_HOUR: "TIMEZONE_HOUR" >
| < TIMEZONE_MINUTE: "TIMEZONE_MINUTE" >
| < TINYINT: "TINYINT" >
@@ -7937,6 +8167,7 @@ SqlPostfixOperator PostfixRowOperator() :
| < TRIM_ARRAY: "TRIM_ARRAY" >
| < TRUE: "TRUE" >
| < TRUNCATE: "TRUNCATE" >
+| < TUESDAY: "TUESDAY" >
| < TUMBLE: "TUMBLE" >
| < TYPE: "TYPE" >
| < UESCAPE: "UESCAPE" >
@@ -7974,7 +8205,9 @@ SqlPostfixOperator PostfixRowOperator() :
| < VERSION: "VERSION" >
| < VERSIONING: "VERSIONING" >
| < VIEW: "VIEW" >
+| < WEDNESDAY: "WEDNESDAY" >
| < WEEK: "WEEK" >
+| < WEEKS: "WEEKS" >
| < WHEN: "WHEN" >
| < WHENEVER: "WHENEVER" >
| < WHERE: "WHERE" >
@@ -8108,6 +8341,8 @@ void NonReservedKeyWord2of3() :
< PREFIXED_STRING_LITERAL: ("_" <CHARSETNAME> | "N") <QUOTED_STRING> >
|
< UNICODE_STRING_LITERAL: "U" "&" <QUOTED_STRING> >
+|
+ < C_STYLE_ESCAPED_STRING_LITERAL: "E" <QUOTE> ( (~["'", "\\"]) | ("\\" ~[]) | "''")* <QUOTE> >
|
< #CHARSETNAME: (["a"-"z","A"-"Z","0"-"9"])
(["a"-"z","A"-"Z","0"-"9",":",".","-","_"])*
diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/validate/FlinkSqlConformance.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/validate/FlinkSqlConformance.java
index a04ce07d1cd..b1235d6a42c 100644
--- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/validate/FlinkSqlConformance.java
+++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/validate/FlinkSqlConformance.java
@@ -166,4 +166,14 @@ public enum FlinkSqlConformance implements SqlConformance {
public SqlLibrary semantics() {
return SqlConformanceEnum.DEFAULT.semantics();
}
+
+ @Override
+ public boolean allowCoercionStringToArray() {
+ return SqlConformanceEnum.DEFAULT.allowCoercionStringToArray();
+ }
+
+ @Override
+ public boolean isValueAllowed() {
+ return true;
+ }
}
diff --git a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java
index 4245ff492cc..c78ae5f2bdf 100644
--- a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java
+++ b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java
@@ -2131,8 +2131,8 @@ class FlinkSqlParserImplTest extends SqlParserTest {
"INSERT INTO `EMPS` "
+ "PARTITION (`X` = 'ab', `Y` = 'bc')\n"
+ "(`X`, `Y`)\n"
- + "(SELECT *\n"
- + "FROM `EMPS`)";
+ + "SELECT *\n"
+ + "FROM `EMPS`";
sql(sql1).ok(expected);
final String sql2 =
"insert into emp\n"
@@ -2148,8 +2148,8 @@ class FlinkSqlParserImplTest extends SqlParserTest {
+ "PARTITION (`EMPNO` = '1', `JOB` = 'job')\n"
+ "(`EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`,"
+ " `COMM`, `DEPTNO`, `SLACKER`)\n"
- + "(SELECT 'nom', 0, TIMESTAMP '1970-01-01 00:00:00', 1, 1, 1, FALSE\n"
- + "FROM (VALUES (ROW('a'))))");
+ + "SELECT 'nom', 0, TIMESTAMP '1970-01-01 00:00:00', 1, 1, 1, FALSE\n"
+ + "FROM (VALUES (ROW('a')))");
final String sql3 =
"insert into empnullables\n"
+ "partition(ename='b')\n"
@@ -2160,8 +2160,8 @@ class FlinkSqlParserImplTest extends SqlParserTest {
"INSERT INTO `EMPNULLABLES` "
+ "PARTITION (`ENAME` = 'b')\n"
+ "(`EMPNO`, `ENAME`)\n"
- + "(SELECT 1\n"
- + "FROM (VALUES (ROW('a'))))");
+ + "SELECT 1\n"
+ + "FROM (VALUES (ROW('a')))");
}
@Test
@@ -2170,8 +2170,8 @@ class FlinkSqlParserImplTest extends SqlParserTest {
"INSERT INTO `emps` "
+ "PARTITION (`x` = 'ab', `y` = 'bc')\n"
+ "(`x`, `y`)\n"
- + "(SELECT *\n"
- + "FROM `EMPS`)";
+ + "SELECT *\n"
+ + "FROM `EMPS`";
sql("insert into \"emps\" "
+ "partition (\"x\"='ab', \"y\"='bc')(\"x\",\"y\") select * from emps")
.ok(expected);
@@ -2183,8 +2183,8 @@ class FlinkSqlParserImplTest extends SqlParserTest {
"INSERT INTO `EMPS` EXTEND (`Z` BOOLEAN) "
+ "PARTITION (`Z` = 'ab')\n"
+ "(`X`, `Y`)\n"
- + "(SELECT *\n"
- + "FROM `EMPS`)";
+ + "SELECT *\n"
+ + "FROM `EMPS`";
sql("insert into emps(z boolean) partition (z='ab') (x,y) select * from emps").ok(expected);
}
@@ -2204,7 +2204,7 @@ class FlinkSqlParserImplTest extends SqlParserTest {
void testInsertOverwrite() {
// non-partitioned
final String sql = "INSERT OVERWRITE myDB.myTbl SELECT * FROM src";
- final String expected = "INSERT OVERWRITE `MYDB`.`MYTBL`\n" + "(SELECT *\n" + "FROM `SRC`)";
+ final String expected = "INSERT OVERWRITE `MYDB`.`MYTBL`\n" + "SELECT *\n" + "FROM `SRC`";
sql(sql).ok(expected);
// partitioned
@@ -2213,8 +2213,8 @@ class FlinkSqlParserImplTest extends SqlParserTest {
"INSERT OVERWRITE `MYTBL` "
+ "PARTITION (`P1` = 'v1', `P2` = 'v2')\n"
+ "\n"
- + "(SELECT *\n"
- + "FROM `SRC`)";
+ + "SELECT *\n"
+ + "FROM `SRC`";
sql(sql1).ok(expected1);
}
@@ -2552,12 +2552,12 @@ class FlinkSqlParserImplTest extends SqlParserTest {
.ok(
"EXECUTE STATEMENT SET BEGIN\n"
+ "INSERT INTO `T1`\n"
- + "(SELECT *\n"
- + "FROM `T2`)\n"
+ + "SELECT *\n"
+ + "FROM `T2`\n"
+ ";\n"
+ "INSERT INTO `T2`\n"
- + "(SELECT *\n"
- + "FROM `T3`)\n"
+ + "SELECT *\n"
+ + "FROM `T3`\n"
+ ";\n"
+ "END");
}
@@ -2568,12 +2568,12 @@ class FlinkSqlParserImplTest extends SqlParserTest {
.ok(
"EXPLAIN STATEMENT SET BEGIN\n"
+ "INSERT INTO `T1`\n"
- + "(SELECT *\n"
- + "FROM `T2`)\n"
+ + "SELECT *\n"
+ + "FROM `T2`\n"
+ ";\n"
+ "INSERT INTO `T2`\n"
- + "(SELECT *\n"
- + "FROM `T3`)\n"
+ + "SELECT *\n"
+ + "FROM `T3`\n"
+ ";\n"
+ "END");
}
@@ -2617,11 +2617,11 @@ class FlinkSqlParserImplTest extends SqlParserTest {
void testExplainUnion() {
String sql = "explain estimated_cost select * from emps union all select * from emps";
String expected =
- "EXPLAIN ESTIMATED_COST (SELECT *\n"
+ "EXPLAIN ESTIMATED_COST SELECT *\n"
+ "FROM `EMPS`\n"
+ "UNION ALL\n"
+ "SELECT *\n"
- + "FROM `EMPS`)";
+ + "FROM `EMPS`";
this.sql(sql).ok(expected);
}
@@ -2681,13 +2681,13 @@ class FlinkSqlParserImplTest extends SqlParserTest {
@Test
void testExplainInsert() {
- String expected = "EXPLAIN INSERT INTO `EMPS1`\n" + "(SELECT *\n" + "FROM `EMPS2`)";
+ String expected = "EXPLAIN INSERT INTO `EMPS1`\n" + "SELECT *\n" + "FROM `EMPS2`";
this.sql("explain plan for insert into emps1 select * from emps2").ok(expected);
}
@Test
void testExecuteInsert() {
- String expected = "EXECUTE INSERT INTO `EMPS1`\n" + "(SELECT *\n" + "FROM `EMPS2`)";
+ String expected = "EXECUTE INSERT INTO `EMPS1`\n" + "SELECT *\n" + "FROM `EMPS2`";
this.sql("execute insert into emps1 select * from emps2").ok(expected);
}
@@ -2705,30 +2705,30 @@ class FlinkSqlParserImplTest extends SqlParserTest {
sql("compile plan './test.json' for insert into t1 select * from t2")
.ok(
"COMPILE PLAN './test.json' FOR INSERT INTO `T1`\n"
- + "(SELECT *\n"
- + "FROM `T2`)");
+ + "SELECT *\n"
+ + "FROM `T2`");
sql("compile plan './test.json' if not exists for insert into t1 select * from t2")
.ok(
"COMPILE PLAN './test.json' IF NOT EXISTS FOR INSERT INTO `T1`\n"
- + "(SELECT *\n"
- + "FROM `T2`)");
+ + "SELECT *\n"
+ + "FROM `T2`");
sql("compile plan 'file:///foo/bar/test.json' if not exists for insert into t1 select * from t2")
.ok(
"COMPILE PLAN 'file:///foo/bar/test.json' IF NOT EXISTS FOR INSERT INTO `T1`\n"
- + "(SELECT *\n"
- + "FROM `T2`)");
+ + "SELECT *\n"
+ + "FROM `T2`");
sql("compile plan './test.json' for statement set "
+ "begin insert into t1 select * from t2; insert into t2 select * from t3; end")
.ok(
"COMPILE PLAN './test.json' FOR STATEMENT SET BEGIN\n"
+ "INSERT INTO `T1`\n"
- + "(SELECT *\n"
- + "FROM `T2`)\n"
+ + "SELECT *\n"
+ + "FROM `T2`\n"
+ ";\n"
+ "INSERT INTO `T2`\n"
- + "(SELECT *\n"
- + "FROM `T3`)\n"
+ + "SELECT *\n"
+ + "FROM `T3`\n"
+ ";\n"
+ "END");
sql("compile plan './test.json' if not exists for statement set "
@@ -2736,12 +2736,12 @@ class FlinkSqlParserImplTest extends SqlParserTest {
.ok(
"COMPILE PLAN './test.json' IF NOT EXISTS FOR STATEMENT SET BEGIN\n"
+ "INSERT INTO `T1`\n"
- + "(SELECT *\n"
- + "FROM `T2`)\n"
+ + "SELECT *\n"
+ + "FROM `T2`\n"
+ ";\n"
+ "INSERT INTO `T2`\n"
- + "(SELECT *\n"
- + "FROM `T3`)\n"
+ + "SELECT *\n"
+ + "FROM `T3`\n"
+ ";\n"
+ "END");
@@ -2750,12 +2750,12 @@ class FlinkSqlParserImplTest extends SqlParserTest {
.ok(
"COMPILE PLAN 'file:///foo/bar/test.json' IF NOT EXISTS FOR STATEMENT SET BEGIN\n"
+ "INSERT INTO `T1`\n"
- + "(SELECT *\n"
- + "FROM `T2`)\n"
+ + "SELECT *\n"
+ + "FROM `T2`\n"
+ ";\n"
+ "INSERT INTO `T2`\n"
- + "(SELECT *\n"
- + "FROM `T3`)\n"
+ + "SELECT *\n"
+ + "FROM `T3`\n"
+ ";\n"
+ "END");
}
@@ -2765,27 +2765,27 @@ class FlinkSqlParserImplTest extends SqlParserTest {
sql("compile and execute plan './test.json' for insert into t1 select * from t2")
.ok(
"COMPILE AND EXECUTE PLAN './test.json' FOR INSERT INTO `T1`\n"
- + "(SELECT *\n"
- + "FROM `T2`)");
+ + "SELECT *\n"
+ + "FROM `T2`");
sql("compile and execute plan './test.json' for statement set "
+ "begin insert into t1 select * from t2; insert into t2 select * from t3; end")
.ok(
"COMPILE AND EXECUTE PLAN './test.json' FOR STATEMENT SET BEGIN\n"
+ "INSERT INTO `T1`\n"
- + "(SELECT *\n"
- + "FROM `T2`)\n"
+ + "SELECT *\n"
+ + "FROM `T2`\n"
+ ";\n"
+ "INSERT INTO `T2`\n"
- + "(SELECT *\n"
- + "FROM `T3`)\n"
+ + "SELECT *\n"
+ + "FROM `T3`\n"
+ ";\n"
+ "END");
sql("compile and execute plan 'file:///foo/bar/test.json' for insert into t1 select * from t2")
.ok(
"COMPILE AND EXECUTE PLAN 'file:///foo/bar/test.json' FOR INSERT INTO `T1`\n"
- + "(SELECT *\n"
- + "FROM `T2`)");
+ + "SELECT *\n"
+ + "FROM `T2`");
}
@Test
@@ -2829,10 +2829,10 @@ class FlinkSqlParserImplTest extends SqlParserTest {
@Test
void testSetReset() {
- sql("SET").ok("SET");
- sql("SET 'test-key' = 'test-value'").ok("SET 'test-key' = 'test-value'");
- sql("RESET").ok("RESET");
- sql("RESET 'test-key'").ok("RESET 'test-key'");
+ sql("SET").same();
+ sql("SET 'test-key' = 'test-value'").same();
+ sql("RESET").same();
+ sql("RESET 'test-key'").same();
}
@Test
@@ -3121,10 +3121,9 @@ class FlinkSqlParserImplTest extends SqlParserTest {
@Test
void testStopJob() {
- sql("STOP JOB 'myjob'").ok("STOP JOB 'myjob'");
- sql("STOP JOB 'myjob' WITH SAVEPOINT").ok("STOP JOB 'myjob' WITH SAVEPOINT");
- sql("STOP JOB 'myjob' WITH SAVEPOINT WITH DRAIN")
- .ok("STOP JOB 'myjob' WITH SAVEPOINT WITH DRAIN");
+ sql("STOP JOB 'myjob'").same();
+ sql("STOP JOB 'myjob' WITH SAVEPOINT").same();
+ sql("STOP JOB 'myjob' WITH SAVEPOINT WITH DRAIN").same();
sql("STOP JOB 'myjob' ^WITH DRAIN^")
.fails("WITH DRAIN could only be used after WITH SAVEPOINT.");
sql("STOP JOB 'myjob' ^WITH DRAIN^ WITH SAVEPOINT")
@@ -3133,7 +3132,7 @@ class FlinkSqlParserImplTest extends SqlParserTest {
@Test
void testDescribeJob() {
- sql("DESCRIBE JOB 'myjob'").ok("DESCRIBE JOB 'myjob'");
+ sql("DESCRIBE JOB 'myjob'").same();
sql("DESC JOB 'myjob'").ok("DESCRIBE JOB 'myjob'");
}
diff --git a/flink-table/flink-table-calcite-bridge/pom.xml b/flink-table/flink-table-calcite-bridge/pom.xml
index 3cb2101c2b9..3c57ecbaab1 100644
--- a/flink-table/flink-table-calcite-bridge/pom.xml
+++ b/flink-table/flink-table-calcite-bridge/pom.xml
@@ -45,23 +45,23 @@ under the License.
<version>${calcite.version}</version>
<exclusions>
<!--
- "mvn dependency:tree" as of Calcite 1.32.0:
- [INFO] +- org.apache.calcite:calcite-core:jar:1.32.0:compile
- [INFO] | +- org.apache.calcite:calcite-linq4j:jar:1.32.0:compile
+ "mvn dependency:tree" as of Calcite 1.33.0:
+ [INFO] +- org.apache.calcite:calcite-core:jar:1.33.0:compile
+ [INFO] | +- org.apache.calcite:calcite-linq4j:jar:1.33.0:compile
[INFO] | +- org.locationtech.jts:jts-core:jar:1.19.0:compile
- [INFO] | +- com.fasterxml.jackson.core:jackson-annotations:jar:2.14.3:compile
- [INFO] | +- org.apache.calcite.avatica:avatica-core:jar:1.22.0:compile
+ [INFO] | +- com.fasterxml.jackson.core:jackson-annotations:jar:2.15.3:compile
+ [INFO] | +- org.apache.calcite.avatica:avatica-core:jar:1.23.0:compile
[INFO] | +- org.apiguardian:apiguardian-api:jar:1.1.2:compile
- [INFO] | +- com.fasterxml.jackson.core:jackson-core:jar:2.14.3:compile
- [INFO] | +- com.fasterxml.jackson.core:jackson-databind:jar:2.14.3:compile
+ [INFO] | +- com.fasterxml.jackson.core:jackson-core:jar:2.15.3:compile
+ [INFO] | +- com.fasterxml.jackson.core:jackson-databind:jar:2.15.3:compile
[INFO] | +- com.jayway.jsonpath:json-path:jar:2.7.0:runtime
[INFO] | | \- net.minidev:json-smart:jar:2.4.7:runtime
[INFO] | | \- net.minidev:accessors-smart:jar:2.4.7:runtime
[INFO] | | \- org.ow2.asm:asm:jar:9.1:runtime
[INFO] | +- commons-codec:commons-codec:jar:1.15:runtime
+ [INFO] | +- org.apache.commons:commons-math3:jar:3.6.1:runtime
[INFO] | \- commons-io:commons-io:jar:2.11.0:runtime
-
Dependencies that are not needed for how we use Calcite right now.
-->
<exclusion>
diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml
index dadcb0da7ef..2458256fb40 100644
--- a/flink-table/flink-table-planner/pom.xml
+++ b/flink-table/flink-table-planner/pom.xml
@@ -77,8 +77,12 @@ under the License.
<artifactId>value-annotations</artifactId>
<version>2.8.8</version>
</dependency>
-
-
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.6.1</version>
+ <optional>${flink.markBundledAsOptional}</optional>
+ </dependency>
<dependency>
<groupId>org.codehaus.janino</groupId>
<artifactId>commons-compiler</artifactId>
@@ -320,6 +324,7 @@ under the License.
<include>com.google.guava:failureaccess</include>
<include>commons-codec:commons-codec</include>
<include>commons-io:commons-io</include>
+ <include>org.apache.commons:commons-math3</include>
<include>org.checkerframework:checker-qual</include>
<!-- flink-table-planner dependencies -->
@@ -348,6 +353,10 @@ under the License.
<pattern>org.apache.commons.io</pattern>
<shadedPattern>org.apache.flink.calcite.shaded.org.apache.commons.io</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>org.apache.commons.math3</pattern>
+ <shadedPattern>org.apache.flink.calcite.shaded.org.apache.commons.math3</shadedPattern>
+ </relocation>
<relocation>
<pattern>org.checkerframework</pattern>
<shadedPattern>org.apache.flink.calcite.shaded.org.checkerframework</shadedPattern>
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rel/logical/LogicalSnapshot.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rel/logical/LogicalSnapshot.java
index f2e974d3dd3..39199fae112 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rel/logical/LogicalSnapshot.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rel/logical/LogicalSnapshot.java
@@ -24,6 +24,7 @@ import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelCollationTraitDef;
import org.apache.calcite.rel.RelDistributionTraitDef;
+import org.apache.calcite.rel.RelInput;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Snapshot;
import org.apache.calcite.rel.hint.RelHint;
@@ -40,12 +41,18 @@ import java.util.List;
* Sub-class of {@link org.apache.calcite.rel.core.Snapshot} not targeted at any particular engine
* or calling convention. The class was copied over because of * CALCITE-4554. *
*
- * <p>Line 106 ~ 117: Calcite only supports timestamp type as period type, but Flink supports both
+ * <p>Line 114 ~ 124: Calcite only supports timestamp type as period type, but Flink supports both
* Timestamp and TimestampLtz. Should be removed once calcite support TimestampLtz as period type.
*/
public class LogicalSnapshot extends Snapshot {
// ~ Constructors -----------------------------------------------------------
+
+ /** Creates a LogicalSnapshot by parsing serialized output. */
+ public LogicalSnapshot(RelInput input) {
+ super(input);
+ }
+
/**
* Creates a LogicalSnapshot.
*
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java
index a11e67ca956..22c8f1e3b62 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java
@@ -27,6 +27,7 @@ import org.apache.calcite.plan.Strong;
import org.apache.calcite.plan.volcano.RelSubset;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.Correlate;
import org.apache.calcite.rel.core.Exchange;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Intersect;
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rex/RexUtil.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rex/RexUtil.java
new file mode 100644
index 00000000000..01dfc2409db
--- /dev/null
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rex/RexUtil.java
@@ -0,0 +1,3176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.rex;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
+import org.apache.calcite.DataContexts;
+import org.apache.calcite.linq4j.function.Predicate1;
+import org.apache.calcite.plan.RelOptPredicateList;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollations;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.Join;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeFamily;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexTableInputRef.RelTableRef;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeFamily;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.type.SqlTypeUtil;
+import org.apache.calcite.sql.validate.SqlValidatorUtil;
+import org.apache.calcite.util.ControlFlowException;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.Litmus;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.RangeSets;
+import org.apache.calcite.util.Sarg;
+import org.apache.calcite.util.Util;
+import org.apache.calcite.util.mapping.Mappings;
+import org.apiguardian.api.API;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Predicate;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Default implementation of {@link org.apache.calcite.rex.RexUtil}, the class was copied over
+ * because of current Calcite way of inferring constants from IS NOT DISTINCT FROM clashes with
+ * filter push down.
+ *
+ * <p>Lines 397 ~ 399, Use Calcite 1.32.0 behavior for {@link RexUtil#gatherConstraints(Class,
+ * RexNode, Map, Set, RexBuilder)}.
+ */
+public class RexUtil {
+
+ /** Executor for a bit of constant reduction. The user can pass in another executor. */
+ public static final RexExecutor EXECUTOR = new RexExecutorImpl(DataContexts.EMPTY);
+
+ private RexUtil() {}
+
+ // ~ Methods ----------------------------------------------------------------
+
+ /**
+ * Returns a guess for the selectivity of an expression.
+ *
+ * @param exp expression of interest, or null for none (implying a selectivity of 1.0)
+ * @return guessed selectivity
+ */
+ public static double getSelectivity(@Nullable RexNode exp) {
+ if ((exp == null) || exp.isAlwaysTrue()) {
+ return 1d;
+ }
+ return 0.1d;
+ }
+
+ /**
+ * Generates a cast from one row type to another.
+ *
+ * @param rexBuilder RexBuilder to use for constructing casts
+ * @param lhsRowType target row type
+ * @param rhsRowType source row type; fields must be 1-to-1 with lhsRowType, in same order
+ * @return cast expressions
+ */
+ public static List<RexNode> generateCastExpressions(
+ RexBuilder rexBuilder, RelDataType lhsRowType, RelDataType rhsRowType) {
+ final List<RelDataTypeField> fieldList = rhsRowType.getFieldList();
+ int n = fieldList.size();
+ assert n == lhsRowType.getFieldCount()
+ : "field count: lhs [" + lhsRowType + "] rhs [" + rhsRowType + "]";
+ List<RexNode> rhsExps = new ArrayList<>();
+ for (RelDataTypeField field : fieldList) {
+ rhsExps.add(rexBuilder.makeInputRef(field.getType(), field.getIndex()));
+ }
+ return generateCastExpressions(rexBuilder, lhsRowType, rhsExps);
+ }
+
+ /**
+ * Generates a cast for a row type.
+ *
+ * @param rexBuilder RexBuilder to use for constructing casts
+ * @param lhsRowType target row type
+ * @param rhsExps expressions to be cast
+ * @return cast expressions
+ */
+ public static List<RexNode> generateCastExpressions(
+ RexBuilder rexBuilder, RelDataType lhsRowType, List<RexNode> rhsExps) {
+ List<RelDataTypeField> lhsFields = lhsRowType.getFieldList();
+ List<RexNode> castExps = new ArrayList<>();
+ for (Pair<RelDataTypeField, RexNode> pair : Pair.zip(lhsFields, rhsExps, true)) {
+ RelDataTypeField lhsField = pair.left;
+ RelDataType lhsType = lhsField.getType();
+ final RexNode rhsExp = pair.right;
+ RelDataType rhsType = rhsExp.getType();
+ if (lhsType.equals(rhsType)) {
+ castExps.add(rhsExp);
+ } else {
+ castExps.add(rexBuilder.makeCast(lhsType, rhsExp));
+ }
+ }
+ return castExps;
+ }
+
+ /**
+ * Returns whether a node represents the NULL value.
+ *
+ * <p>Examples:
+ *
+ * <ul>
+ * <li>For {@link org.apache.calcite.rex.RexLiteral} Unknown, returns false.
+ * <li>For <code>CAST(NULL AS <i>type</i>)</code>, returns true if <code>
+ * allowCast</code> is true, false otherwise.
+ * <li>For <code>CAST(CAST(NULL AS <i>type</i>) AS <i>type</i>))</code>, returns false.
+ * </ul>
+ */
+ public static boolean isNullLiteral(RexNode node, boolean allowCast) {
+ if (node instanceof RexLiteral) {
+ RexLiteral literal = (RexLiteral) node;
+ if (literal.getTypeName() == SqlTypeName.NULL) {
+ assert null == literal.getValue();
+ return true;
+ } else {
+ // We don't regard UNKNOWN -- SqlLiteral(null,Boolean) -- as
+ // NULL.
+ return false;
+ }
+ }
+ if (allowCast) {
+ if (node.isA(SqlKind.CAST)) {
+ RexCall call = (RexCall) node;
+ if (isNullLiteral(call.operands.get(0), false)) {
+ // node is "CAST(NULL as type)"
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns whether a node represents the NULL value or a series of nested {@code CAST(NULL AS
+ * type)} calls. For example: <code>isNull(CAST(CAST(NULL as INTEGER) AS VARCHAR(1)))</code>
+ * returns {@code true}.
+ */
+ public static boolean isNull(RexNode expr) {
+ switch (expr.getKind()) {
+ case LITERAL:
+ return ((RexLiteral) expr).getValue2() == null;
+ case CAST:
+ return isNull(((RexCall) expr).operands.get(0));
+ default:
+ return false;
+ }
+ }
+
+ /** Returns whether a node represents a {@link SqlTypeName#SYMBOL} literal. */
+ public static boolean isSymbolLiteral(RexNode expr) {
+ switch (expr.getKind()) {
+ case LITERAL:
+ return ((RexLiteral) expr).getTypeName() == SqlTypeName.SYMBOL;
+ case CAST:
+ return isSymbolLiteral(((RexCall) expr).operands.get(0));
+ default:
+ return false;
+ }
+ }
+
+ /**
+ * Returns whether a node represents a literal.
+ *
+ * <p>Examples:
+ *
+ * <ul>
+ * <li>For <code>CAST(literal AS <i>type</i>)</code>, returns true if <code>
+ * allowCast</code> is true, false otherwise.
+ * <li>For <code>CAST(CAST(literal AS <i>type</i>) AS <i>type</i>))</code>, returns false.
+ * </ul>
+ *
+ * @param node The node, never null.
+ * @param allowCast whether to regard CAST(literal) as a literal
+ * @return Whether the node is a literal
+ */
+ public static boolean isLiteral(RexNode node, boolean allowCast) {
+ assert node != null;
+ if (node.isA(SqlKind.LITERAL)) {
+ return true;
+ }
+ if (allowCast) {
+ if (node.isA(SqlKind.CAST)) {
+ RexCall call = (RexCall) node;
+ if (isLiteral(call.operands.get(0), false)) {
+ // node is "CAST(literal as type)"
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns whether every expression in a list is a literal.
+ *
+ * @param expressionOperands list of expressions to check
+ * @return true if every expression from the specified list is literal.
+ */
+ public static boolean allLiterals(List<RexNode> expressionOperands) {
+ for (RexNode rexNode : expressionOperands) {
+ if (!isLiteral(rexNode, true)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Returns whether a node represents an input reference or field access.
+ *
+ * @param node The node, never null.
+ * @param allowCast whether to regard CAST(x) as true
+ * @return Whether the node is a reference or access
+ */
+ public static boolean isReferenceOrAccess(RexNode node, boolean allowCast) {
+ assert node != null;
+ if (node instanceof RexInputRef || node instanceof RexFieldAccess) {
+ return true;
+ }
+ if (allowCast) {
+ if (node.isA(SqlKind.CAST)) {
+ RexCall call = (RexCall) node;
+ return isReferenceOrAccess(call.operands.get(0), false);
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns whether an expression is a cast just for the purposes of nullability, not changing
+ * any other aspect of the type.
+ */
+ public static boolean isNullabilityCast(RelDataTypeFactory typeFactory, RexNode node) {
+ switch (node.getKind()) {
+ case CAST:
+ final RexCall call = (RexCall) node;
+ final RexNode arg0 = call.getOperands().get(0);
+ return SqlTypeUtil.equalSansNullability(
+ typeFactory, arg0.getType(), call.getType());
+ default:
+ break;
+ }
+ return false;
+ }
+
+ /**
+ * Removes any casts that change nullability but not type.
+ *
+ * <p>For example, {@code CAST(1 = 0 AS BOOLEAN)} becomes {@code 1 = 0}.
+ */
+ public static RexNode removeNullabilityCast(RelDataTypeFactory typeFactory, RexNode node) {
+ while (isNullabilityCast(typeFactory, node)) {
+ node = ((RexCall) node).operands.get(0);
+ }
+ return node;
+ }
+
+ /**
+ * Removes any casts.
+ *
+ * <p>For example, {@code CAST('1' AS INTEGER)} becomes {@code '1'}.
+ */
+ public static RexNode removeCast(RexNode e) {
+ for (; ; ) {
+ switch (e.getKind()) {
+ case CAST:
+ e = ((RexCall) e).operands.get(0);
+ break;
+ default:
+ return e;
+ }
+ }
+ }
+
+ /**
+ * Creates a map containing each (e, constant) pair that occurs within a predicate list.
+ *
+ * @param clazz Class of expression that is considered constant
+ * @param rexBuilder Rex builder
+ * @param predicates Predicate list
+ * @param <C> what to consider a constant: {@link RexLiteral} to use a narrow definition of
+ * constant, or {@link RexNode} to use {@link RexUtil#isConstant(RexNode)}
+ * @return Map from values to constants
+ */
+ public static <C extends RexNode> ImmutableMap<RexNode, C> predicateConstants(
+ Class<C> clazz, RexBuilder rexBuilder, List<RexNode> predicates) {
+ // We cannot use an ImmutableMap.Builder here. If there are multiple entries
+ // with the same key (e.g. "WHERE deptno = 1 AND deptno = 2"), it doesn't
+ // matter which we take, so the latter will replace the former.
+ // The basic idea is to find all the pairs of RexNode = RexLiteral
+ // (1) If 'predicates' contain a non-EQUALS, we bail out.
+ // (2) It is OK if a RexNode is equal to the same RexLiteral several times,
+ // (e.g. "WHERE deptno = 1 AND deptno = 1")
+ // (3) It will return false if there are inconsistent constraints (e.g.
+ // "WHERE deptno = 1 AND deptno = 2")
+ final Map<RexNode, C> map = new HashMap<>();
+ final Set<RexNode> excludeSet = new HashSet<>();
+ for (RexNode predicate : predicates) {
+ gatherConstraints(clazz, predicate, map, excludeSet, rexBuilder);
+ }
+ final ImmutableMap.Builder<RexNode, C> builder = ImmutableMap.builder();
+ for (Map.Entry<RexNode, C> entry : map.entrySet()) {
+ RexNode rexNode = entry.getKey();
+ if (!overlap(rexNode, excludeSet)) {
+ builder.put(rexNode, entry.getValue());
+ }
+ }
+ return builder.build();
+ }
+
+ private static boolean overlap(RexNode rexNode, Set<RexNode> set) {
+ if (rexNode instanceof RexCall) {
+ for (RexNode r : ((RexCall) rexNode).getOperands()) {
+ if (overlap(r, set)) {
+ return true;
+ }
+ }
+ return false;
+ } else {
+ return set.contains(rexNode);
+ }
+ }
+
+ /** Tries to decompose the RexNode which is a RexCall into non-literal RexNodes. */
+ private static void decompose(Set<RexNode> set, RexNode rexNode) {
+ if (rexNode instanceof RexCall) {
+ for (RexNode r : ((RexCall) rexNode).getOperands()) {
+ decompose(set, r);
+ }
+ } else if (!(rexNode instanceof RexLiteral)) {
+ set.add(rexNode);
+ }
+ }
+
+ private static <C extends RexNode> void gatherConstraints(
+ Class<C> clazz,
+ RexNode predicate,
+ Map<RexNode, C> map,
+ Set<RexNode> excludeSet,
+ RexBuilder rexBuilder) {
+ final RexNode left;
+ final RexNode right;
+ switch (predicate.getKind()) {
+ case EQUALS:
+ // FLINK BEGIN MODIFICATION
+ // case IS_NOT_DISTINCT_FROM:
+ // FLINK END MODIFICATION
+ left = ((RexCall) predicate).getOperands().get(0);
+ right = ((RexCall) predicate).getOperands().get(1);
+ break;
+
+ case IS_NULL:
+ left = ((RexCall) predicate).getOperands().get(0);
+ if (!left.getType().isNullable()) {
+ // There's no sense in inferring $0=null when $0 is not nullable
+ return;
+ }
+ right = rexBuilder.makeNullLiteral(left.getType());
+ break;
+
+ default:
+ decompose(excludeSet, predicate);
+ return;
+ }
+ // Note that literals are immutable too, and they can only be compared
+ // through values.
+ gatherConstraint(clazz, left, right, map, excludeSet, rexBuilder);
+ gatherConstraint(clazz, right, left, map, excludeSet, rexBuilder);
+ }
+
+ private static <C extends RexNode> void gatherConstraint(
+ Class<C> clazz,
+ RexNode left,
+ RexNode right,
+ Map<RexNode, C> map,
+ Set<RexNode> excludeSet,
+ RexBuilder rexBuilder) {
+ if (!clazz.isInstance(right)) {
+ return;
+ }
+ if (!isConstant(right)) {
+ return;
+ }
+ C constant = clazz.cast(right);
+ if (excludeSet.contains(left)) {
+ return;
+ }
+ final C existedValue = map.get(left);
+ if (existedValue == null) {
+ switch (left.getKind()) {
+ case CAST:
+ // Convert "CAST(c) = literal" to "c = literal", as long as it is a
+ // widening cast.
+ final RexNode operand = ((RexCall) left).getOperands().get(0);
+ if (canAssignFrom(
+ left.getType(), operand.getType(), rexBuilder.getTypeFactory())) {
+ final RexNode castRight = rexBuilder.makeCast(operand.getType(), constant);
+ if (castRight instanceof RexLiteral) {
+ left = operand;
+ constant = clazz.cast(castRight);
+ }
+ }
+ break;
+ default:
+ break;
+ }
+ map.put(left, constant);
+ } else {
+ if (existedValue instanceof RexLiteral
+ && constant instanceof RexLiteral
+ && !Objects.equals(
+ ((RexLiteral) existedValue).getValue(),
+ ((RexLiteral) constant).getValue())) {
+ // we found conflicting values, e.g. left = 10 and left = 20
+ map.remove(left);
+ excludeSet.add(left);
+ }
+ }
+ }
+
+ /**
+ * Returns whether a value of {@code type2} can be assigned to a variable of {@code type1}.
+ *
+ * <p>For example:
+ *
+ * <ul>
+ * <li>{@code canAssignFrom(BIGINT, TINYINT)} returns {@code true}
+ * <li>{@code canAssignFrom(TINYINT, BIGINT)} returns {@code false}
+ * <li>{@code canAssignFrom(BIGINT, VARCHAR)} returns {@code false}
+ * </ul>
+ */
+ private static boolean canAssignFrom(
+ RelDataType type1, RelDataType type2, RelDataTypeFactory typeFactory) {
+ final SqlTypeName name1 = type1.getSqlTypeName();
+ final SqlTypeName name2 = type2.getSqlTypeName();
+ final RelDataType type1Final = type1;
+ SqlTypeFamily family =
+ requireNonNull(
+ name1.getFamily(),
+ () ->
+ "SqlTypeFamily is null for type "
+ + type1Final
+ + ", SqlTypeName "
+ + name1);
+ if (family == name2.getFamily()) {
+ switch (family) {
+ case NUMERIC:
+ if (SqlTypeUtil.isExactNumeric(type1) && SqlTypeUtil.isExactNumeric(type2)) {
+ int precision1;
+ int scale1;
+ if (name1 == SqlTypeName.DECIMAL) {
+ type1 = typeFactory.decimalOf(type1);
+ precision1 = type1.getPrecision();
+ scale1 = type1.getScale();
+ } else {
+ precision1 = typeFactory.getTypeSystem().getMaxPrecision(name1);
+ scale1 = typeFactory.getTypeSystem().getMaxScale(name1);
+ }
+ int precision2;
+ int scale2;
+ if (name2 == SqlTypeName.DECIMAL) {
+ type2 = typeFactory.decimalOf(type2);
+ precision2 = type2.getPrecision();
+ scale2 = type2.getScale();
+ } else {
+ precision2 = typeFactory.getTypeSystem().getMaxPrecision(name2);
+ scale2 = typeFactory.getTypeSystem().getMaxScale(name2);
+ }
+ return precision1 >= precision2 && scale1 >= scale2;
+ } else if (SqlTypeUtil.isApproximateNumeric(type1)
+ && SqlTypeUtil.isApproximateNumeric(type2)) {
+ return type1.getPrecision() >= type2.getPrecision()
+ && type1.getScale() >= type2.getScale();
+ }
+ break;
+ default:
+ // getPrecision() will return:
+ // - number of decimal digits for fractional seconds for datetime types
+ // - length in characters for character types
+ // - length in bytes for binary types
+ // - RelDataType.PRECISION_NOT_SPECIFIED (-1) if not applicable for this type
+ return type1.getPrecision() >= type2.getPrecision();
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns the number of nodes (including leaves) in a list of expressions.
+ *
+ * @see RexNode#nodeCount()
+ */
+ public static int nodeCount(List<? extends RexNode> nodes) {
+ return nodeCount(0, nodes);
+ }
+
+ static int nodeCount(int n, List<? extends RexNode> nodes) {
+ for (RexNode operand : nodes) {
+ n += operand.nodeCount();
+ }
+ return n;
+ }
+
+ /** Returns a visitor that finds nodes of a given {@link SqlKind}. */
+ public static RexFinder find(final SqlKind kind) {
+ return new RexFinder() {
+ @Override
+ public Void visitCall(RexCall call) {
+ if (call.getKind() == kind) {
+ throw Util.FoundOne.NULL;
+ }
+ return super.visitCall(call);
+ }
+ };
+ }
+
+ /** Returns a visitor that finds nodes of given {@link SqlKind}s. */
+ public static RexFinder find(final Set<SqlKind> kinds) {
+ return new RexFinder() {
+ @Override
+ public Void visitCall(RexCall call) {
+ if (kinds.contains(call.getKind())) {
+ throw Util.FoundOne.NULL;
+ }
+ return super.visitCall(call);
+ }
+ };
+ }
+
+ /** Returns a visitor that finds a particular {@link RexInputRef}. */
+ public static RexFinder find(final RexInputRef ref) {
+ return new RexFinder() {
+ @Override
+ public Void visitInputRef(RexInputRef inputRef) {
+ if (ref.equals(inputRef)) {
+ throw Util.FoundOne.NULL;
+ }
+ return super.visitInputRef(inputRef);
+ }
+ };
+ }
+
+ /** Expands all the calls to {@link SqlStdOperatorTable#SEARCH} in an expression. */
+ public static RexNode expandSearch(
+ RexBuilder rexBuilder, @Nullable RexProgram program, RexNode node) {
+ return expandSearch(rexBuilder, program, node, -1);
+ }
+
+ /**
+ * Expands calls to {@link SqlStdOperatorTable#SEARCH} whose complexity is greater than {@code
+ * maxComplexity} in an expression.
+ */
+ public static RexNode expandSearch(
+ RexBuilder rexBuilder, @Nullable RexProgram program, RexNode node, int maxComplexity) {
+ return node.accept(searchShuttle(rexBuilder, program, maxComplexity));
+ }
+
+ /**
+ * Creates a shuttle that expands calls to {@link SqlStdOperatorTable#SEARCH}.
+ *
+ * <p>If {@code maxComplexity} is non-negative, a {@link Sarg} whose complexity is greater than
+ * {@code maxComplexity} is retained (not expanded); this gives a means to simplify simple
+ * expressions such as {@code x IS NULL} or {@code x > 10} while keeping more complex
+ * expressions such as {@code x IN (3, 5, 7) OR x IS NULL} as a Sarg.
+ */
+ public static RexShuttle searchShuttle(
+ RexBuilder rexBuilder, @Nullable RexProgram program, int maxComplexity) {
+ return new SearchExpandingShuttle(program, rexBuilder, maxComplexity);
+ }
+
+ @SuppressWarnings("BetaApi")
+ public static <C extends Comparable<C>> RexNode sargRef(
+ RexBuilder rexBuilder,
+ RexNode ref,
+ Sarg<C> sarg,
+ RelDataType type,
+ RexUnknownAs unknownAs) {
+ if (sarg.isAll() || sarg.isNone()) {
+ return simpleSarg(rexBuilder, ref, sarg, unknownAs);
+ }
+ final List<RexNode> orList = new ArrayList<>();
+ if (sarg.nullAs == RexUnknownAs.TRUE && unknownAs == RexUnknownAs.UNKNOWN) {
+ orList.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, ref));
+ }
+ if (sarg.isPoints()) {
+ // Generate 'ref = value1 OR ... OR ref = valueN'
+ sarg.rangeSet
+ .asRanges()
+ .forEach(
+ range ->
+ orList.add(
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.EQUALS,
+ ref,
+ rexBuilder.makeLiteral(
+ range.lowerEndpoint(),
+ type,
+ true,
+ true))));
+ } else if (sarg.isComplementedPoints()) {
+ // Generate 'ref <> value1 AND ... AND ref <> valueN'
+ final List<RexNode> list =
+ sarg.rangeSet.complement().asRanges().stream()
+ .map(
+ range ->
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.NOT_EQUALS,
+ ref,
+ rexBuilder.makeLiteral(
+ range.lowerEndpoint(),
+ type,
+ true,
+ true)))
+ .collect(Util.toImmutableList());
+ orList.add(composeConjunction(rexBuilder, list));
+ } else {
+ final RangeSets.Consumer<C> consumer = new RangeToRex<>(ref, orList, rexBuilder, type);
+ RangeSets.forEach(sarg.rangeSet, consumer);
+ }
+ RexNode node = composeDisjunction(rexBuilder, orList);
+ if (sarg.nullAs == RexUnknownAs.FALSE && unknownAs == RexUnknownAs.UNKNOWN) {
+ node =
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.AND,
+ rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, ref),
+ node);
+ }
+ return node;
+ }
+
+ /** Expands an 'all' or 'none' sarg. */
+ public static <C extends Comparable<C>> RexNode simpleSarg(
+ RexBuilder rexBuilder, RexNode ref, Sarg<C> sarg, RexUnknownAs unknownAs) {
+ assert sarg.isAll() || sarg.isNone();
+ final RexUnknownAs nullAs = sarg.nullAs == RexUnknownAs.UNKNOWN ? unknownAs : sarg.nullAs;
+ if (sarg.isAll()) {
+ switch (nullAs) {
+ case TRUE:
+ return rexBuilder.makeLiteral(true);
+ case FALSE:
+ return rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, ref);
+ case UNKNOWN:
+ // "x IS NOT NULL OR UNKNOWN"
+ return rexBuilder.makeCall(
+ SqlStdOperatorTable.OR,
+ rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, ref),
+ rexBuilder.makeNullLiteral(
+ rexBuilder.typeFactory.createSqlType(SqlTypeName.BOOLEAN)));
+ }
+ }
+ if (sarg.isNone()) {
+ switch (nullAs) {
+ case TRUE:
+ return rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, ref);
+ case FALSE:
+ return rexBuilder.makeLiteral(false);
+ case UNKNOWN:
+ // "CASE WHEN x IS NULL THEN UNKNOWN ELSE FALSE END", or "x <> x"
+ return rexBuilder.makeCall(SqlStdOperatorTable.NOT_EQUALS, ref, ref);
+ }
+ }
+ throw new AssertionError();
+ }
+
+ private static RexNode deref(@Nullable RexProgram program, RexNode node) {
+ while (node instanceof RexLocalRef) {
+ node = requireNonNull(program, "program").getExprList().get(((RexLocalRef) node).index);
+ }
+ return node;
+ }
+
+ /** Walks over an expression and determines whether it is constant. */
+ static class ConstantFinder implements RexVisitor<Boolean> {
+ static final ConstantFinder INSTANCE = new ConstantFinder();
+
+ @Override
+ public Boolean visitLiteral(RexLiteral literal) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitInputRef(RexInputRef inputRef) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitLocalRef(RexLocalRef localRef) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitOver(RexOver over) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitSubQuery(RexSubQuery subQuery) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitTableInputRef(RexTableInputRef ref) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitPatternFieldRef(RexPatternFieldRef fieldRef) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitCorrelVariable(RexCorrelVariable correlVariable) {
+ // Correlating variables change when there is an internal restart.
+ // Not good enough for our purposes.
+ return false;
+ }
+
+ @Override
+ public Boolean visitDynamicParam(RexDynamicParam dynamicParam) {
+ // Dynamic parameters are constant WITHIN AN EXECUTION, so that's
+ // good enough.
+ return true;
+ }
+
+ @Override
+ public Boolean visitCall(RexCall call) {
+ // Constant if operator meets the following conditions:
+ // 1. It is deterministic;
+ // 2. All its operands are constant.
+ return call.getOperator().isDeterministic()
+ && RexVisitorImpl.visitArrayAnd(this, call.getOperands());
+ }
+
+ @Override
+ public Boolean visitRangeRef(RexRangeRef rangeRef) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitFieldAccess(RexFieldAccess fieldAccess) {
+ // "<expr>.FIELD" is constant iff "<expr>" is constant.
+ return fieldAccess.getReferenceExpr().accept(this);
+ }
+ }
+
+ /**
+ * Returns whether node is made up of constants.
+ *
+ * @param node Node to inspect
+ * @return true if node is made up of constants, false otherwise
+ */
+ public static boolean isConstant(RexNode node) {
+ return node.accept(ConstantFinder.INSTANCE);
+ }
+
+ /**
+ * Returns whether a given expression is deterministic.
+ *
+ * @param e Expression
+ * @return true if tree result is deterministic, false otherwise
+ */
+ public static boolean isDeterministic(RexNode e) {
+ try {
+ RexVisitor<Void> visitor =
+ new RexVisitorImpl<Void>(true) {
+ @Override
+ public Void visitCall(RexCall call) {
+ if (!call.getOperator().isDeterministic()) {
+ throw Util.FoundOne.NULL;
+ }
+ return super.visitCall(call);
+ }
+ };
+ e.accept(visitor);
+ return true;
+ } catch (Util.FoundOne ex) {
+ Util.swallow(ex, null);
+ return false;
+ }
+ }
+
+ public static List<RexNode> retainDeterministic(List<RexNode> list) {
+ List<RexNode> conjuctions = new ArrayList<>();
+ for (RexNode x : list) {
+ if (isDeterministic(x)) {
+ conjuctions.add(x);
+ }
+ }
+ return conjuctions;
+ }
+
+ /**
+ * Returns whether a given node contains a RexCall with a specified operator.
+ *
+ * @param operator Operator to look for
+ * @param node a RexNode tree
+ */
+ public static @Nullable RexCall findOperatorCall(final SqlOperator operator, RexNode node) {
+ try {
+ RexVisitor<Void> visitor =
+ new RexVisitorImpl<Void>(true) {
+ @Override
+ public Void visitCall(RexCall call) {
+ if (call.getOperator().equals(operator)) {
+ throw new Util.FoundOne(call);
+ }
+ return super.visitCall(call);
+ }
+ };
+ node.accept(visitor);
+ return null;
+ } catch (Util.FoundOne e) {
+ Util.swallow(e, null);
+ return (RexCall) e.getNode();
+ }
+ }
+
+ /**
+ * Returns whether a given tree contains any {link RexInputRef} nodes.
+ *
+ * @param node a RexNode tree
+ */
+ public static boolean containsInputRef(RexNode node) {
+ try {
+ RexVisitor<Void> visitor =
+ new RexVisitorImpl<Void>(true) {
+ @Override
+ public Void visitInputRef(RexInputRef inputRef) {
+ throw new Util.FoundOne(inputRef);
+ }
+ };
+ node.accept(visitor);
+ return false;
+ } catch (Util.FoundOne e) {
+ Util.swallow(e, null);
+ return true;
+ }
+ }
+
+ /**
+ * Returns whether a given tree contains any {@link org.apache.calcite.rex.RexFieldAccess}
+ * nodes.
+ *
+ * @param node a RexNode tree
+ */
+ public static boolean containsFieldAccess(RexNode node) {
+ try {
+ RexVisitor<Void> visitor =
+ new RexVisitorImpl<Void>(true) {
+ @Override
+ public Void visitFieldAccess(RexFieldAccess fieldAccess) {
+ throw new Util.FoundOne(fieldAccess);
+ }
+ };
+ node.accept(visitor);
+ return false;
+ } catch (Util.FoundOne e) {
+ Util.swallow(e, null);
+ return true;
+ }
+ }
+
+ /**
+ * Determines whether a {@link RexCall} requires decimal expansion. It usually requires
+ * expansion if it has decimal operands.
+ *
+ * <p>Exceptions to this rule are:
+ *
+ * <ul>
+ * <li>isNull doesn't require expansion
+ * <li>It's okay to cast decimals to and from char types
+ * <li>It's okay to cast nulls as decimals
+ * <li>Casts require expansion if their return type is decimal
+ * <li>Reinterpret casts can handle a decimal operand
+ * </ul>
+ *
+ * @param expr expression possibly in need of expansion
+ * @param recurse whether to check nested calls
+ * @return whether the expression requires expansion
+ */
+ public static boolean requiresDecimalExpansion(RexNode expr, boolean recurse) {
+ if (!(expr instanceof RexCall)) {
+ return false;
+ }
+ RexCall call = (RexCall) expr;
+
+ boolean localCheck = true;
+ switch (call.getKind()) {
+ case REINTERPRET:
+ case IS_NULL:
+ localCheck = false;
+ break;
+ case CAST:
+ RelDataType lhsType = call.getType();
+ RelDataType rhsType = call.operands.get(0).getType();
+ if (rhsType.getSqlTypeName() == SqlTypeName.NULL) {
+ return false;
+ }
+ if (SqlTypeUtil.inCharFamily(lhsType) || SqlTypeUtil.inCharFamily(rhsType)) {
+ localCheck = false;
+ } else if (SqlTypeUtil.isDecimal(lhsType) && (lhsType != rhsType)) {
+ return true;
+ }
+ break;
+ default:
+ localCheck = call.getOperator().requiresDecimalExpansion();
+ }
+
+ if (localCheck) {
+ if (SqlTypeUtil.isDecimal(call.getType())) {
+ // NOTE jvs 27-Mar-2007: Depending on the type factory, the
+ // result of a division may be decimal, even though both inputs
+ // are integer.
+ return true;
+ }
+ for (int i = 0; i < call.operands.size(); i++) {
+ if (SqlTypeUtil.isDecimal(call.operands.get(i).getType())) {
+ return true;
+ }
+ }
+ }
+ return recurse && requiresDecimalExpansion(call.operands, true);
+ }
+
+ /** Determines whether any operand of a set requires decimal expansion. */
+ public static boolean requiresDecimalExpansion(List<RexNode> operands, boolean recurse) {
+ for (RexNode operand : operands) {
+ if (operand instanceof RexCall) {
+ RexCall call = (RexCall) operand;
+ if (requiresDecimalExpansion(call, recurse)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns whether a {@link RexProgram} contains expressions which require decimal expansion.
+ */
+ public static boolean requiresDecimalExpansion(RexProgram program, boolean recurse) {
+ final List<RexNode> exprList = program.getExprList();
+ for (RexNode expr : exprList) {
+ if (requiresDecimalExpansion(expr, recurse)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static boolean canReinterpretOverflow(RexCall call) {
+ assert call.isA(SqlKind.REINTERPRET) : "call is not a reinterpret";
+ return call.operands.size() > 1;
+ }
+
+ /** Returns whether an array of expressions has any common sub-expressions. */
+ public static boolean containNoCommonExprs(List<RexNode> exprs, Litmus litmus) {
+ final ExpressionNormalizer visitor = new ExpressionNormalizer(false);
+ for (RexNode expr : exprs) {
+ try {
+ expr.accept(visitor);
+ } catch (ExpressionNormalizer.SubExprExistsException e) {
+ Util.swallow(e, null);
+ return litmus.fail(null);
+ }
+ }
+ return litmus.succeed();
+ }
+
+ /**
+ * Returns whether an array of expressions contains no forward references. That is, if
+ * expression #i contains a {@link RexInputRef} referencing field i or greater.
+ *
+ * @param exprs Array of expressions
+ * @param inputRowType Input row type
+ * @param litmus What to do if an error is detected (there is a forward reference)
+ * @return Whether there is a forward reference
+ */
+ public static boolean containNoForwardRefs(
+ List<RexNode> exprs, RelDataType inputRowType, Litmus litmus) {
+ final ForwardRefFinder visitor = new ForwardRefFinder(inputRowType);
+ for (int i = 0; i < exprs.size(); i++) {
+ RexNode expr = exprs.get(i);
+ visitor.setLimit(i); // field cannot refer to self or later field
+ try {
+ expr.accept(visitor);
+ } catch (ForwardRefFinder.IllegalForwardRefException e) {
+ Util.swallow(e, null);
+ return litmus.fail("illegal forward reference in {}", expr);
+ }
+ }
+ return litmus.succeed();
+ }
+
+ /**
+ * Returns whether an array of exp contains no aggregate function calls whose arguments are not
+ * {@link RexInputRef}s.
+ *
+ * @param exprs Expressions
+ * @param litmus Whether to assert if there is such a function call
+ */
+ static boolean containNoNonTrivialAggs(List<RexNode> exprs, Litmus litmus) {
+ for (RexNode expr : exprs) {
+ if (expr instanceof RexCall) {
+ RexCall rexCall = (RexCall) expr;
+ if (rexCall.getOperator() instanceof SqlAggFunction) {
+ for (RexNode operand : rexCall.operands) {
+ if (!(operand instanceof RexLocalRef) && !(operand instanceof RexLiteral)) {
+ return litmus.fail("contains non trivial agg: {}", operand);
+ }
+ }
+ }
+ }
+ }
+ return litmus.succeed();
+ }
+
+ /**
+ * Returns whether a list of expressions contains complex expressions, that is, a call whose
+ * arguments are not {@link RexVariable} (or a subtype such as {@link RexInputRef}) or {@link
+ * RexLiteral}.
+ */
+ public static boolean containComplexExprs(List<RexNode> exprs) {
+ for (RexNode expr : exprs) {
+ if (expr instanceof RexCall) {
+ for (RexNode operand : ((RexCall) expr).operands) {
+ if (!isAtomic(operand)) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns whether any of the given expression trees contains a {link RexTableInputRef} node.
+ *
+ * @param nodes a list of RexNode trees
+ * @return true if at least one was found, otherwise false
+ */
+ public static boolean containsTableInputRef(List<RexNode> nodes) {
+ for (RexNode e : nodes) {
+ if (containsTableInputRef(e) != null) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns whether a given tree contains any {link RexTableInputRef} nodes.
+ *
+ * @param node a RexNode tree
+ * @return first such node found or null if it there is no such node
+ */
+ public static @Nullable RexTableInputRef containsTableInputRef(RexNode node) {
+ try {
+ RexVisitor<Void> visitor =
+ new RexVisitorImpl<Void>(true) {
+ @Override
+ public Void visitTableInputRef(RexTableInputRef inputRef) {
+ throw new Util.FoundOne(inputRef);
+ }
+ };
+ node.accept(visitor);
+ return null;
+ } catch (Util.FoundOne e) {
+ Util.swallow(e, null);
+ return (RexTableInputRef) e.getNode();
+ }
+ }
+
+ public static boolean isAtomic(RexNode expr) {
+ return (expr instanceof RexLiteral) || (expr instanceof RexVariable);
+ }
+
+ /**
+ * Returns whether a {@link RexNode node} is a {@link RexCall call} to a given {@link
+ * SqlOperator operator}.
+ */
+ public static boolean isCallTo(RexNode expr, SqlOperator op) {
+ return (expr instanceof RexCall) && (((RexCall) expr).getOperator() == op);
+ }
+
+ /**
+ * Creates a record type with anonymous field names.
+ *
+ * @param typeFactory Type factory
+ * @param exprs Expressions
+ * @return Record type
+ */
+ public static RelDataType createStructType(
+ RelDataTypeFactory typeFactory, final List<RexNode> exprs) {
+ return createStructType(typeFactory, exprs, null, null);
+ }
+
+ /**
+ * Creates a record type with specified field names.
+ *
+ * <p>The array of field names may be null, or any of the names within it can be null. We
+ * recommend using explicit names where possible, because it makes it much easier to figure out
+ * the intent of fields when looking at planner output.
+ *
+ * @param typeFactory Type factory
+ * @param exprs Expressions
+ * @param names Field names, may be null, or elements may be null
+ * @param suggester Generates alternative names if {@code names} is not null and its elements
+ * are not unique
+ * @return Record type
+ */
+ public static RelDataType createStructType(
+ RelDataTypeFactory typeFactory,
+ final List<? extends RexNode> exprs,
+ @Nullable List<? extends @Nullable String> names,
+ SqlValidatorUtil.Suggester suggester) {
+ if (names != null && suggester != null) {
+ names =
+ SqlValidatorUtil.uniquify(
+ names, suggester, typeFactory.getTypeSystem().isSchemaCaseSensitive());
+ }
+ final RelDataTypeFactory.Builder builder = typeFactory.builder();
+ for (int i = 0; i < exprs.size(); i++) {
+ String name;
+ if (names == null || (name = names.get(i)) == null) {
+ name = "$f" + i;
+ }
+ builder.add(name, exprs.get(i).getType());
+ }
+ return builder.build();
+ }
+
+ @Deprecated // to be removed before 2.0
+ public static RelDataType createStructType(
+ RelDataTypeFactory typeFactory,
+ final List<? extends RexNode> exprs,
+ List<String> names) {
+ return createStructType(typeFactory, exprs, names, null);
+ }
+
+ /**
+ * Returns whether the type of an array of expressions is compatible with a struct type.
+ *
+ * @param exprs Array of expressions
+ * @param type Type
+ * @param litmus What to do if an error is detected (there is a mismatch)
+ * @return Whether every expression has the same type as the corresponding member of the struct
+ * type
+ * @see RelOptUtil#eq(String, RelDataType, String, RelDataType, org.apache.calcite.util.Litmus)
+ */
+ public static boolean compatibleTypes(List<RexNode> exprs, RelDataType type, Litmus litmus) {
+ final List<RelDataTypeField> fields = type.getFieldList();
+ if (exprs.size() != fields.size()) {
+ return litmus.fail("rowtype mismatches expressions");
+ }
+ for (int i = 0; i < fields.size(); i++) {
+ final RelDataType exprType = exprs.get(i).getType();
+ final RelDataType fieldType = fields.get(i).getType();
+ if (!RelOptUtil.eq("type1", exprType, "type2", fieldType, litmus)) {
+ return litmus.fail(null);
+ }
+ }
+ return litmus.succeed();
+ }
+
+ /**
+ * Creates a key for {@link RexNode} which is the same as another key of another RexNode only if
+ * the two have both the same type and textual representation. For example, "10" integer and
+ * "10" bigint result in different keys.
+ */
+ public static Pair<RexNode, String> makeKey(RexNode expr) {
+ return Pair.of(expr, expr.getType().getFullTypeString());
+ }
+
+ /**
+ * Returns whether the leading edge of a given array of expressions is wholly {@link
+ * RexInputRef} objects with types corresponding to the underlying datatype.
+ */
+ public static boolean containIdentity(
+ List<? extends RexNode> exprs, RelDataType rowType, Litmus litmus) {
+ final List<RelDataTypeField> fields = rowType.getFieldList();
+ if (exprs.size() < fields.size()) {
+ return litmus.fail("exprs/rowType length mismatch");
+ }
+ for (int i = 0; i < fields.size(); i++) {
+ if (!(exprs.get(i) instanceof RexInputRef)) {
+ return litmus.fail("expr[{}] is not a RexInputRef", i);
+ }
+ RexInputRef inputRef = (RexInputRef) exprs.get(i);
+ if (inputRef.getIndex() != i) {
+ return litmus.fail("expr[{}] has ordinal {}", i, inputRef.getIndex());
+ }
+ if (!RelOptUtil.eq(
+ "type1", exprs.get(i).getType(), "type2", fields.get(i).getType(), litmus)) {
+ return litmus.fail(null);
+ }
+ }
+ return litmus.succeed();
+ }
+
+ /** Returns whether a list of expressions projects the incoming fields. */
+ public static boolean isIdentity(List<? extends RexNode> exps, RelDataType inputRowType) {
+ return inputRowType.getFieldCount() == exps.size()
+ && containIdentity(exps, inputRowType, Litmus.IGNORE);
+ }
+
+ /** As {@link #composeConjunction(RexBuilder, Iterable, boolean)} but never returns null. */
+ public static RexNode composeConjunction(
+ RexBuilder rexBuilder, Iterable<? extends @Nullable RexNode> nodes) {
+ final RexNode e = composeConjunction(rexBuilder, nodes, false);
+ return requireNonNull(e, "e");
+ }
+
+ /**
+ * Converts a collection of expressions into an AND. If there are zero expressions, returns
+ * TRUE. If there is one expression, returns just that expression. If any of the expressions are
+ * FALSE, returns FALSE. Removes expressions that always evaluate to TRUE. Returns null only if
+ * {@code nullOnEmpty} and expression is TRUE.
+ */
+ public static @Nullable RexNode composeConjunction(
+ RexBuilder rexBuilder,
+ Iterable<? extends @Nullable RexNode> nodes,
+ boolean nullOnEmpty) {
+ ImmutableList<RexNode> list = flattenAnd(nodes);
+ switch (list.size()) {
+ case 0:
+ return nullOnEmpty ? null : rexBuilder.makeLiteral(true);
+ case 1:
+ return list.get(0);
+ default:
+ if (containsFalse(list)) {
+ return rexBuilder.makeLiteral(false);
+ }
+ return rexBuilder.makeCall(SqlStdOperatorTable.AND, list);
+ }
+ }
+
+ /**
+ * Flattens a list of AND nodes.
+ *
+ * <p>Treats null nodes as literal TRUE (i.e. ignores them).
+ */
+ public static ImmutableList<RexNode> flattenAnd(Iterable<? extends @Nullable RexNode> nodes) {
+ if (nodes instanceof Collection && ((Collection) nodes).isEmpty()) {
+ // Optimize common case
+ return ImmutableList.of();
+ }
+ final ImmutableList.Builder<RexNode> builder = ImmutableList.builder();
+ final Set<RexNode> set = new HashSet<>(); // to eliminate duplicates
+ for (RexNode node : nodes) {
+ if (node != null) {
+ addAnd(builder, set, node);
+ }
+ }
+ return builder.build();
+ }
+
+ private static void addAnd(
+ ImmutableList.Builder<RexNode> builder, Set<RexNode> digests, RexNode node) {
+ switch (node.getKind()) {
+ case AND:
+ for (RexNode operand : ((RexCall) node).getOperands()) {
+ addAnd(builder, digests, operand);
+ }
+ return;
+ default:
+ if (!node.isAlwaysTrue() && digests.add(node)) {
+ builder.add(node);
+ }
+ }
+ }
+
+ /**
+ * Converts a collection of expressions into an OR. If there are zero expressions, returns
+ * FALSE. If there is one expression, returns just that expression. If any of the expressions
+ * are TRUE, returns TRUE. Removes expressions that always evaluate to FALSE. Flattens
+ * expressions that are ORs.
+ */
+ public static RexNode composeDisjunction(
+ RexBuilder rexBuilder, Iterable<? extends RexNode> nodes) {
+ final RexNode e = composeDisjunction(rexBuilder, nodes, false);
+ return requireNonNull(e, "e");
+ }
+
+ /**
+ * Converts a collection of expressions into an OR, optionally returning null if the list is
+ * empty.
+ */
+ public static @Nullable RexNode composeDisjunction(
+ RexBuilder rexBuilder, Iterable<? extends RexNode> nodes, boolean nullOnEmpty) {
+ ImmutableList<RexNode> list = flattenOr(nodes);
+ switch (list.size()) {
+ case 0:
+ return nullOnEmpty ? null : rexBuilder.makeLiteral(false);
+ case 1:
+ return list.get(0);
+ default:
+ if (containsTrue(list)) {
+ return rexBuilder.makeLiteral(true);
+ }
+ return rexBuilder.makeCall(SqlStdOperatorTable.OR, list);
+ }
+ }
+
+ /** Flattens a list of OR nodes. */
+ public static ImmutableList<RexNode> flattenOr(Iterable<? extends RexNode> nodes) {
+ if (nodes instanceof Collection && ((Collection) nodes).isEmpty()) {
+ // Optimize common case
+ return ImmutableList.of();
+ }
+ final ImmutableList.Builder<RexNode> builder = ImmutableList.builder();
+ final Set<RexNode> set = new HashSet<>(); // to eliminate duplicates
+ for (RexNode node : nodes) {
+ addOr(builder, set, node);
+ }
+ return builder.build();
+ }
+
+ private static void addOr(
+ ImmutableList.Builder<RexNode> builder, Set<RexNode> set, RexNode node) {
+ switch (node.getKind()) {
+ case OR:
+ for (RexNode operand : ((RexCall) node).getOperands()) {
+ addOr(builder, set, operand);
+ }
+ return;
+ default:
+ if (!node.isAlwaysFalse() && set.add(node)) {
+ builder.add(node);
+ }
+ }
+ }
+
+ /**
+ * Applies a mapping to a collation list.
+ *
+ * @param mapping Mapping
+ * @param collationList Collation list
+ * @return collation list with mapping applied to each field
+ */
+ public static List<RelCollation> apply(
+ Mappings.TargetMapping mapping, List<RelCollation> collationList) {
+ final List<RelCollation> newCollationList = new ArrayList<>();
+ for (RelCollation collation : collationList) {
+ final List<RelFieldCollation> newFieldCollationList = new ArrayList<>();
+ for (RelFieldCollation fieldCollation : collation.getFieldCollations()) {
+ final RelFieldCollation newFieldCollation = apply(mapping, fieldCollation);
+ if (newFieldCollation == null) {
+ // This field is not mapped. Stop here. The leading edge
+ // of the collation is still valid (although it's useless
+ // if it's empty).
+ break;
+ }
+ newFieldCollationList.add(newFieldCollation);
+ }
+ // Truncation to collations to their leading edge creates empty
+ // and duplicate collations. Ignore these.
+ if (!newFieldCollationList.isEmpty()) {
+ final RelCollation newCollation = RelCollations.of(newFieldCollationList);
+ if (!newCollationList.contains(newCollation)) {
+ newCollationList.add(newCollation);
+ }
+ }
+ }
+
+ // REVIEW: There might be redundant collations in the list. For example,
+ // in {(x), (x, y)}, (x) is redundant because it is a leading edge of
+ // another collation in the list. Could remove redundant collations.
+
+ return newCollationList;
+ }
+
+ /**
+ * Applies a mapping to a collation.
+ *
+ * @param mapping Mapping
+ * @param collation Collation
+ * @return collation with mapping applied
+ */
+ public static RelCollation apply(Mappings.TargetMapping mapping, RelCollation collation) {
+ List<RelFieldCollation> fieldCollations =
+ applyFields(mapping, collation.getFieldCollations());
+ return fieldCollations.equals(collation.getFieldCollations())
+ ? collation
+ : RelCollations.of(fieldCollations);
+ }
+
+ /**
+ * Applies a mapping to a field collation.
+ *
+ * <p>If the field is not mapped, returns null.
+ *
+ * @param mapping Mapping
+ * @param fieldCollation Field collation
+ * @return collation with mapping applied
+ */
+ public static @Nullable RelFieldCollation apply(
+ Mappings.TargetMapping mapping, RelFieldCollation fieldCollation) {
+ final int target = mapping.getTargetOpt(fieldCollation.getFieldIndex());
+ if (target < 0) {
+ return null;
+ }
+ return fieldCollation.withFieldIndex(target);
+ }
+
+ /**
+ * Applies a mapping to a list of field collations.
+ *
+ * @param mapping Mapping
+ * @param fieldCollations Field collations
+ * @return collations with mapping applied
+ */
+ public static List<RelFieldCollation> applyFields(
+ Mappings.TargetMapping mapping, List<RelFieldCollation> fieldCollations) {
+ final List<RelFieldCollation> newFieldCollations = new ArrayList<>();
+ for (RelFieldCollation fieldCollation : fieldCollations) {
+ RelFieldCollation newFieldCollation = apply(mapping, fieldCollation);
+ if (newFieldCollation == null) {
+ break;
+ }
+ newFieldCollations.add(newFieldCollation);
+ }
+ return newFieldCollations;
+ }
+
+ /** Applies a mapping to an expression. */
+ public static RexNode apply(Mappings.TargetMapping mapping, RexNode node) {
+ return node.accept(RexPermuteInputsShuttle.of(mapping));
+ }
+
+ /** Applies a mapping to an iterable over expressions. */
+ public static List<RexNode> apply(
+ Mappings.TargetMapping mapping, Iterable<? extends RexNode> nodes) {
+ return RexPermuteInputsShuttle.of(mapping).visitList(nodes);
+ }
+
+ /**
+ * Applies a shuttle to an array of expressions. Creates a copy first.
+ *
+ * @param shuttle Shuttle
+ * @param exprs Array of expressions
+ */
+ public static <T extends RexNode> T[] apply(RexVisitor<T> shuttle, T[] exprs) {
+ T[] newExprs = exprs.clone();
+ for (int i = 0; i < newExprs.length; i++) {
+ final RexNode expr = newExprs[i];
+ if (expr != null) {
+ newExprs[i] = expr.accept(shuttle);
+ }
+ }
+ return newExprs;
+ }
+
+ /**
+ * Applies a visitor to an array of expressions and, if specified, a single expression.
+ *
+ * @param visitor Visitor
+ * @param exprs Array of expressions
+ * @param expr Single expression, may be null
+ */
+ public static void apply(RexVisitor<Void> visitor, RexNode[] exprs, @Nullable RexNode expr) {
+ for (RexNode e : exprs) {
+ e.accept(visitor);
+ }
+ if (expr != null) {
+ expr.accept(visitor);
+ }
+ }
+
+ /**
+ * Applies a visitor to a list of expressions and, if specified, a single expression.
+ *
+ * @param visitor Visitor
+ * @param exprs List of expressions
+ * @param expr Single expression, may be null
+ */
+ public static void apply(
+ RexVisitor<Void> visitor, List<? extends RexNode> exprs, @Nullable RexNode expr) {
+ for (RexNode e : exprs) {
+ e.accept(visitor);
+ }
+ if (expr != null) {
+ expr.accept(visitor);
+ }
+ }
+
+ /**
+ * Flattens an expression.
+ *
+ * <p>Returns the same expression if it is already flat.
+ */
+ public static RexNode flatten(RexBuilder rexBuilder, RexNode node) {
+ if (node instanceof RexCall) {
+ RexCall call = (RexCall) node;
+ final SqlOperator op = call.getOperator();
+ final List<RexNode> flattenedOperands = flatten(call.getOperands(), op);
+ if (!isFlat(call.getOperands(), op)) {
+ return rexBuilder.makeCall(call.getType(), op, flattenedOperands);
+ }
+ }
+ return node;
+ }
+
+ /**
+ * Converts a list of operands into a list that is flat with respect to the given operator. The
+ * operands are assumed to be flat already.
+ */
+ public static List<RexNode> flatten(List<? extends RexNode> exprs, SqlOperator op) {
+ if (isFlat(exprs, op)) {
+ //noinspection unchecked
+ return (List) exprs;
+ }
+ final List<RexNode> list = new ArrayList<>();
+ flattenRecurse(list, exprs, op);
+ return list;
+ }
+
+ /**
+ * Returns whether a call to {@code op} with {@code exprs} as arguments would be considered
+ * "flat".
+ *
+ * <p>For example, {@code isFlat([w, AND[x, y], z, AND)} returns false;
+ *
+ * <p>{@code isFlat([w, x, y, z], AND)} returns true.
+ */
+ private static boolean isFlat(List<? extends RexNode> exprs, final SqlOperator op) {
+ return !isAssociative(op)
+ || !exists(exprs, (Predicate1<RexNode>) expr -> isCallTo(expr, op));
+ }
+
+ /**
+ * Returns false if the expression can be optimized by flattening calls to an associative
+ * operator such as AND and OR.
+ */
+ public static boolean isFlat(RexNode expr) {
+ if (!(expr instanceof RexCall)) {
+ return true;
+ }
+ final RexCall call = (RexCall) expr;
+ return isFlat(call.getOperands(), call.getOperator())
+ && all(call.getOperands(), RexUtil::isFlat);
+ }
+
+ private static void flattenRecurse(
+ List<RexNode> list, List<? extends RexNode> exprs, SqlOperator op) {
+ for (RexNode expr : exprs) {
+ if (expr instanceof RexCall && ((RexCall) expr).getOperator() == op) {
+ flattenRecurse(list, ((RexCall) expr).getOperands(), op);
+ } else {
+ list.add(expr);
+ }
+ }
+ }
+
+ /**
+ * Returns whether the input is a 'loss-less' cast, that is, a cast from which the original
+ * value of the field can be certainly recovered.
+ *
+ * <p>For instance, int → bigint is loss-less (as you can cast back to int without loss of
+ * information), but bigint → int is not loss-less.
+ *
+ * <p>The implementation of this method does not return false positives. However, it is not
+ * complete.
+ *
+ * @param node input node to verify if it represents a loss-less cast
+ * @return true iff the node is a loss-less cast
+ */
+ public static boolean isLosslessCast(RexNode node) {
+ if (!node.isA(SqlKind.CAST)) {
+ return false;
+ }
+ return isLosslessCast(((RexCall) node).getOperands().get(0).getType(), node.getType());
+ }
+
+ /**
+ * Returns whether the conversion from {@code source} to {@code target} type is a 'loss-less'
+ * cast, that is, a cast from which the original value of the field can be certainly recovered.
+ *
+ * <p>For instance, int → bigint is loss-less (as you can cast back to int without loss of
+ * information), but bigint → int is not loss-less.
+ *
+ * <p>The implementation of this method does not return false positives. However, it is not
+ * complete.
+ *
+ * @param source source type
+ * @param target target type
+ * @return true iff the conversion is a loss-less cast
+ */
+ @API(since = "1.22", status = API.Status.EXPERIMENTAL)
+ public static boolean isLosslessCast(RelDataType source, RelDataType target) {
+ final SqlTypeName sourceSqlTypeName = source.getSqlTypeName();
+ final SqlTypeName targetSqlTypeName = target.getSqlTypeName();
+ // 1) Both INT numeric types
+ if (SqlTypeFamily.INTEGER.getTypeNames().contains(sourceSqlTypeName)
+ && SqlTypeFamily.INTEGER.getTypeNames().contains(targetSqlTypeName)) {
+ return targetSqlTypeName.compareTo(sourceSqlTypeName) >= 0;
+ }
+ // 2) Both CHARACTER types: it depends on the precision (length)
+ if (SqlTypeFamily.CHARACTER.getTypeNames().contains(sourceSqlTypeName)
+ && SqlTypeFamily.CHARACTER.getTypeNames().contains(targetSqlTypeName)) {
+ return targetSqlTypeName.compareTo(sourceSqlTypeName) >= 0
+ && source.getPrecision() <= target.getPrecision();
+ }
+ // 3) From NUMERIC family to CHARACTER family: it depends on the precision/scale
+ if (sourceSqlTypeName.getFamily() == SqlTypeFamily.NUMERIC
+ && targetSqlTypeName.getFamily() == SqlTypeFamily.CHARACTER) {
+ int sourceLength = source.getPrecision() + 1; // include sign
+ if (source.getScale() != -1 && source.getScale() != 0) {
+ sourceLength += source.getScale() + 1; // include decimal mark
+ }
+ return target.getPrecision() >= sourceLength;
+ }
+ // Return FALSE by default
+ return false;
+ }
+
+ /**
+ * Converts an expression to conjunctive normal form (CNF).
+ *
+ * <p>The following expression is in CNF:
+ *
+ * <blockquote>
+ *
+ * (a OR b) AND (c OR d)
+ *
+ * </blockquote>
+ *
+ * <p>The following expression is not in CNF:
+ *
+ * <blockquote>
+ *
+ * (a AND b) OR c
+ *
+ * </blockquote>
+ *
+ * <p>but can be converted to CNF:
+ *
+ * <blockquote>
+ *
+ * (a OR c) AND (b OR c)
+ *
+ * </blockquote>
+ *
+ * <p>The following expression is not in CNF:
+ *
+ * <blockquote>
+ *
+ * NOT (a OR NOT b)
+ *
+ * </blockquote>
+ *
+ * <p>but can be converted to CNF by applying de Morgan's theorem:
+ *
+ * <blockquote>
+ *
+ * NOT a AND b
+ *
+ * </blockquote>
+ *
+ * <p>Expressions not involving AND, OR or NOT at the top level are in CNF.
+ */
+ public static RexNode toCnf(RexBuilder rexBuilder, RexNode rex) {
+ return new CnfHelper(rexBuilder, -1).toCnf(rex);
+ }
+
+ /**
+ * Similar to {@link #toCnf(RexBuilder, RexNode)}; however, it lets you specify a threshold in
+ * the number of nodes that can be created out of the conversion.
+ *
+ * <p>If the number of resulting nodes exceeds that threshold, stops conversion and returns the
+ * original expression.
+ *
+ * <p>If the threshold is negative it is ignored.
+ *
+ * <p>Leaf nodes in the expression do not count towards the threshold.
+ */
+ public static RexNode toCnf(RexBuilder rexBuilder, int maxCnfNodeCount, RexNode rex) {
+ return new CnfHelper(rexBuilder, maxCnfNodeCount).toCnf(rex);
+ }
+
+ /**
+ * Converts an expression to disjunctive normal form (DNF).
+ *
+ * <p>DNF: It is a form of logical formula which is disjunction of conjunctive clauses.
+ *
+ * <p>All logical formulas can be converted into DNF.
+ *
+ * <p>The following expression is in DNF:
+ *
+ * <blockquote>
+ *
+ * (a AND b) OR (c AND d)
+ *
+ * </blockquote>
+ *
+ * <p>The following expression is not in CNF:
+ *
+ * <blockquote>
+ *
+ * (a OR b) AND c
+ *
+ * </blockquote>
+ *
+ * <p>but can be converted to DNF:
+ *
+ * <blockquote>
+ *
+ * (a AND c) OR (b AND c)
+ *
+ * </blockquote>
+ *
+ * <p>The following expression is not in CNF:
+ *
+ * <blockquote>
+ *
+ * NOT (a OR NOT b)
+ *
+ * </blockquote>
+ *
+ * <p>but can be converted to DNF by applying de Morgan's theorem:
+ *
+ * <blockquote>
+ *
+ * NOT a AND b
+ *
+ * </blockquote>
+ *
+ * <p>Expressions not involving AND, OR or NOT at the top level are in DNF.
+ */
+ public static RexNode toDnf(RexBuilder rexBuilder, RexNode rex) {
+ return new DnfHelper(rexBuilder).toDnf(rex);
+ }
+
+ /**
+ * Returns whether an operator is associative. AND is associative, which means that "(x AND y)
+ * and z" is equivalent to "x AND (y AND z)". We might well flatten the tree, and write "AND(x,
+ * y, z)".
+ */
+ private static boolean isAssociative(SqlOperator op) {
+ return op.getKind() == SqlKind.AND || op.getKind() == SqlKind.OR;
+ }
+
+ /** Returns whether there is an element in {@code list} for which {@code predicate} is true. */
+ public static <E> boolean exists(List<? extends E> list, Predicate1<E> predicate) {
+ for (E e : list) {
+ if (predicate.apply(e)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** Returns whether {@code predicate} is true for all elements of {@code list}. */
+ public static <E> boolean all(List<? extends E> list, Predicate1<E> predicate) {
+ for (E e : list) {
+ if (!predicate.apply(e)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /** Shifts every {@link RexInputRef} in an expression by {@code offset}. */
+ public static RexNode shift(RexNode node, final int offset) {
+ if (offset == 0) {
+ return node;
+ }
+ return node.accept(new RexShiftShuttle(offset));
+ }
+
+ /** Shifts every {@link RexInputRef} in an expression by {@code offset}. */
+ public static List<RexNode> shift(Iterable<RexNode> nodes, int offset) {
+ return new RexShiftShuttle(offset).visitList(nodes);
+ }
+
+ /**
+ * Shifts every {@link RexInputRef} in an expression higher than {@code start} by {@code
+ * offset}.
+ */
+ public static RexNode shift(RexNode node, final int start, final int offset) {
+ return node.accept(
+ new RexShuttle() {
+ @Override
+ public RexNode visitInputRef(RexInputRef input) {
+ final int index = input.getIndex();
+ if (index < start) {
+ return input;
+ }
+ return new RexInputRef(index + offset, input.getType());
+ }
+ });
+ }
+
+ /**
+ * Creates an equivalent version of a node where common factors among ORs are pulled up.
+ *
+ * <p>For example,
+ *
+ * <blockquote>
+ *
+ * (a AND b) OR (a AND c AND d)
+ *
+ * </blockquote>
+ *
+ * <p>becomes
+ *
+ * <blockquote>
+ *
+ * a AND (b OR (c AND d))
+ *
+ * </blockquote>
+ *
+ * <p>Note that this result is not in CNF (see {@link #toCnf(RexBuilder, RexNode)}) because
+ * there is an AND inside an OR.
+ *
+ * <p>This form is useful if, say, {@code a} contains columns from only the left-hand side of a
+ * join, and can be pushed to the left input.
+ *
+ * @param rexBuilder Rex builder
+ * @param node Expression to transform
+ * @return Equivalent expression with common factors pulled up
+ */
+ public static RexNode pullFactors(RexBuilder rexBuilder, RexNode node) {
+ return new CnfHelper(rexBuilder, -1).pull(node);
+ }
+
+ @Deprecated // to be removed before 2.0
+ public static List<RexNode> fixUp(
+ final RexBuilder rexBuilder, List<RexNode> nodes, final RelDataType rowType) {
+ final List<RelDataType> typeList = RelOptUtil.getFieldTypeList(rowType);
+ return fixUp(rexBuilder, nodes, typeList);
+ }
+
+ /**
+ * Fixes up the type of all {@link RexInputRef}s in an expression to match differences in
+ * nullability.
+ *
+ * <p>Such differences in nullability occur when expressions are moved through outer joins.
+ *
+ * <p>Throws if there any greater inconsistencies of type.
+ */
+ public static List<RexNode> fixUp(
+ final RexBuilder rexBuilder, List<RexNode> nodes, final List<RelDataType> fieldTypes) {
+ return new FixNullabilityShuttle(rexBuilder, fieldTypes).apply(nodes);
+ }
+
+ /** Transforms a list of expressions into a list of their types. */
+ public static List<RelDataType> types(List<? extends RexNode> nodes) {
+ return Util.transform(nodes, RexNode::getType);
+ }
+
+ public static List<RelDataTypeFamily> families(List<RelDataType> types) {
+ return Util.transform(types, RelDataType::getFamily);
+ }
+
+ /**
+ * Removes all expressions from a list that are equivalent to a given expression. Returns
+ * whether any were removed.
+ */
+ public static boolean removeAll(List<RexNode> targets, RexNode e) {
+ int count = 0;
+ Iterator<RexNode> iterator = targets.iterator();
+ while (iterator.hasNext()) {
+ RexNode next = iterator.next();
+ if (next.equals(e)) {
+ ++count;
+ iterator.remove();
+ }
+ }
+ return count > 0;
+ }
+
+ /**
+ * Returns whether two {@link RexNode}s are structurally equal.
+ *
+ * <p>This method considers structure, not semantics. 'x < y' is not equivalent to 'y >
+ * x'.
+ */
+ @Deprecated // use e1.equals(e2)
+ public static boolean eq(RexNode e1, RexNode e2) {
+ return e1 == e2 || e1.toString().equals(e2.toString());
+ }
+
+ /**
+ * Simplifies a boolean expression, always preserving its type and its nullability.
+ *
+ * <p>This is useful if you are simplifying expressions in a {@link Project}.
+ *
+ * @deprecated Use {@link RexSimplify#simplifyPreservingType(RexNode)}, which allows you to
+ * specify an {@link RexExecutor}.
+ */
+ @Deprecated // to be removed before 2.0
+ public static RexNode simplifyPreservingType(RexBuilder rexBuilder, RexNode e) {
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, EXECUTOR)
+ .simplifyPreservingType(e);
+ }
+
+ /**
+ * Simplifies a boolean expression, leaving UNKNOWN values as UNKNOWN, and using the default
+ * executor.
+ *
+ * @deprecated Create a {@link RexSimplify}, then call its {@link RexSimplify#simplify(RexNode,
+ * RexUnknownAs)} method.
+ */
+ @Deprecated // to be removed before 2.0
+ public static RexNode simplify(RexBuilder rexBuilder, RexNode e) {
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, EXECUTOR).simplify(e);
+ }
+
+ /**
+ * Simplifies a boolean expression, using the default executor.
+ *
+ * <p>In particular:
+ *
+ * <ul>
+ * <li>{@code simplify(x = 1 AND y = 2 AND NOT x = 1)} returns {@code y = 2}
+ * <li>{@code simplify(x = 1 AND FALSE)} returns {@code FALSE}
+ * </ul>
+ *
+ * <p>If the expression is a predicate in a WHERE clause, UNKNOWN values have the same effect as
+ * FALSE. In situations like this, specify {@code unknownAsFalse = true}, so and we can switch
+ * from 3-valued logic to simpler 2-valued logic and make more optimizations.
+ *
+ * @param rexBuilder Rex builder
+ * @param e Expression to simplify
+ * @param unknownAsFalse Whether to convert UNKNOWN values to FALSE
+ * @deprecated Create a {@link RexSimplify}, then call its {@link RexSimplify#simplify(RexNode,
+ * RexUnknownAs)} method.
+ */
+ @Deprecated // to be removed before 2.0
+ public static RexNode simplify(RexBuilder rexBuilder, RexNode e, boolean unknownAsFalse) {
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, EXECUTOR)
+ .simplifyUnknownAs(e, RexUnknownAs.falseIf(unknownAsFalse));
+ }
+
+ /**
+ * Simplifies a conjunction of boolean expressions.
+ *
+ * @deprecated Use {@link RexSimplify#simplifyAnds(Iterable, RexUnknownAs)}.
+ */
+ @Deprecated // to be removed before 2.0
+ public static RexNode simplifyAnds(RexBuilder rexBuilder, Iterable<? extends RexNode> nodes) {
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, EXECUTOR)
+ .simplifyAnds(nodes, RexUnknownAs.UNKNOWN);
+ }
+
+ @Deprecated // to be removed before 2.0
+ public static RexNode simplifyAnds(
+ RexBuilder rexBuilder, Iterable<? extends RexNode> nodes, boolean unknownAsFalse) {
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, EXECUTOR)
+ .simplifyAnds(nodes, RexUnknownAs.falseIf(unknownAsFalse));
+ }
+
+ /** Negates a logical expression by adding or removing a NOT. */
+ public static RexNode not(RexNode e) {
+ switch (e.getKind()) {
+ case NOT:
+ return ((RexCall) e).getOperands().get(0);
+ default:
+ return addNot(e);
+ }
+ }
+
+ private static RexNode addNot(RexNode e) {
+ return new RexCall(e.getType(), SqlStdOperatorTable.NOT, ImmutableList.of(e));
+ }
+
+ @API(since = "1.27.0", status = API.Status.EXPERIMENTAL)
+ public static SqlOperator op(SqlKind kind) {
+ switch (kind) {
+ case IS_FALSE:
+ return SqlStdOperatorTable.IS_FALSE;
+ case IS_TRUE:
+ return SqlStdOperatorTable.IS_TRUE;
+ case IS_UNKNOWN:
+ return SqlStdOperatorTable.IS_UNKNOWN;
+ case IS_NULL:
+ return SqlStdOperatorTable.IS_NULL;
+ case IS_NOT_FALSE:
+ return SqlStdOperatorTable.IS_NOT_FALSE;
+ case IS_NOT_TRUE:
+ return SqlStdOperatorTable.IS_NOT_TRUE;
+ case IS_NOT_NULL:
+ return SqlStdOperatorTable.IS_NOT_NULL;
+ case IS_DISTINCT_FROM:
+ return SqlStdOperatorTable.IS_DISTINCT_FROM;
+ case IS_NOT_DISTINCT_FROM:
+ return SqlStdOperatorTable.IS_NOT_DISTINCT_FROM;
+ case EQUALS:
+ return SqlStdOperatorTable.EQUALS;
+ case NOT_EQUALS:
+ return SqlStdOperatorTable.NOT_EQUALS;
+ case LESS_THAN:
+ return SqlStdOperatorTable.LESS_THAN;
+ case GREATER_THAN:
+ return SqlStdOperatorTable.GREATER_THAN;
+ case LESS_THAN_OR_EQUAL:
+ return SqlStdOperatorTable.LESS_THAN_OR_EQUAL;
+ case GREATER_THAN_OR_EQUAL:
+ return SqlStdOperatorTable.GREATER_THAN_OR_EQUAL;
+ case AND:
+ return SqlStdOperatorTable.AND;
+ case OR:
+ return SqlStdOperatorTable.OR;
+ case COALESCE:
+ return SqlStdOperatorTable.COALESCE;
+ default:
+ throw new AssertionError(kind);
+ }
+ }
+
+ @Deprecated // to be removed before 2.0
+ public static RexNode simplifyAnd(RexBuilder rexBuilder, RexCall e, boolean unknownAsFalse) {
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, EXECUTOR)
+ .simplifyAnd(e, RexUnknownAs.falseIf(unknownAsFalse));
+ }
+
+ @Deprecated // to be removed before 2.0
+ public static RexNode simplifyAnd2(
+ RexBuilder rexBuilder, List<RexNode> terms, List<RexNode> notTerms) {
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, EXECUTOR)
+ .simplifyAnd2(terms, notTerms);
+ }
+
+ @Deprecated // to be removed before 2.0
+ public static RexNode simplifyAnd2ForUnknownAsFalse(
+ RexBuilder rexBuilder, List<RexNode> terms, List<RexNode> notTerms) {
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, EXECUTOR)
+ .simplifyAnd2ForUnknownAsFalse(terms, notTerms);
+ }
+
+ public static @Nullable RexNode negate(RexBuilder rexBuilder, RexCall call) {
+ switch (call.getKind()) {
+ case EQUALS:
+ case NOT_EQUALS:
+ case LESS_THAN:
+ case GREATER_THAN:
+ case LESS_THAN_OR_EQUAL:
+ case GREATER_THAN_OR_EQUAL:
+ final SqlOperator op = op(call.getKind().negateNullSafe());
+ return rexBuilder.makeCall(op, call.getOperands());
+ default:
+ break;
+ }
+ return null;
+ }
+
+ public static @Nullable RexNode invert(RexBuilder rexBuilder, RexCall call) {
+ switch (call.getKind()) {
+ case EQUALS:
+ case NOT_EQUALS:
+ case LESS_THAN:
+ case GREATER_THAN:
+ case LESS_THAN_OR_EQUAL:
+ case GREATER_THAN_OR_EQUAL:
+ final SqlOperator op = requireNonNull(call.getOperator().reverse());
+ return rexBuilder.makeCall(op, Lists.reverse(call.getOperands()));
+ default:
+ return null;
+ }
+ }
+
+ @Deprecated // to be removed before 2.0
+ public static RexNode simplifyOr(RexBuilder rexBuilder, RexCall call) {
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, EXECUTOR)
+ .simplifyUnknownAs(call, RexUnknownAs.UNKNOWN);
+ }
+
+ @Deprecated // to be removed before 2.0
+ public static RexNode simplifyOrs(RexBuilder rexBuilder, List<RexNode> terms) {
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, EXECUTOR)
+ .simplifyUnknownAs(
+ RexUtil.composeDisjunction(rexBuilder, terms), RexUnknownAs.UNKNOWN);
+ }
+
+ /** Creates the expression {@code e1 AND NOT notTerm1 AND NOT notTerm2 ...}. */
+ public static RexNode andNot(RexBuilder rexBuilder, RexNode e, RexNode... notTerms) {
+ return andNot(rexBuilder, e, Arrays.asList(notTerms));
+ }
+
+ /**
+ * Creates the expression {@code e1 AND NOT notTerm1 AND NOT notTerm2 ...}.
+ *
+ * <p>Examples:
+ *
+ * <ul>
+ * <li>andNot(p) returns "p"
+ * <li>andNot(p, n1, n2) returns "p AND NOT n1 AND NOT n2"
+ * <li>andNot(x = 10, x = 20, y = 30, x = 30) returns "x = 10 AND NOT (y = 30)"
+ * </ul>
+ */
+ public static RexNode andNot(
+ final RexBuilder rexBuilder, RexNode e, Iterable<? extends RexNode> notTerms) {
+ // If "e" is of the form "x = literal", remove all "x = otherLiteral"
+ // terms from notTerms.
+ switch (e.getKind()) {
+ case EQUALS:
+ final RexCall call = (RexCall) e;
+ if (call.getOperands().get(1) instanceof RexLiteral) {
+ notTerms =
+ Util.filter(
+ notTerms,
+ e2 -> {
+ switch (e2.getKind()) {
+ case EQUALS:
+ RexCall call2 = (RexCall) e2;
+ if (call2.getOperands()
+ .get(0)
+ .equals(call.getOperands().get(0))
+ && call2.getOperands().get(1)
+ instanceof RexLiteral
+ && !call.getOperands()
+ .get(1)
+ .equals(
+ call2.getOperands()
+ .get(1))) {
+ return false;
+ }
+ break;
+ default:
+ break;
+ }
+ return true;
+ });
+ }
+ break;
+ default:
+ break;
+ }
+ return composeConjunction(
+ rexBuilder,
+ Iterables.concat(
+ ImmutableList.of(e), Util.transform(notTerms, e2 -> not(rexBuilder, e2))));
+ }
+
+ /**
+ * Returns whether a given operand of a CASE expression is a predicate.
+ *
+ * <p>A switched case (CASE x WHEN x1 THEN v1 ... ELSE e END) has an even number of arguments
+ * and odd-numbered arguments are predicates.
+ *
+ * <p>A condition case (CASE WHEN p1 THEN v1 ... ELSE e END) has an odd number of arguments and
+ * even-numbered arguments are predicates, except for the last argument.
+ */
+ public static boolean isCasePredicate(RexCall call, int i) {
+ assert call.getKind() == SqlKind.CASE;
+ return i < call.operands.size() - 1 && (call.operands.size() - i) % 2 == 1;
+ }
+
+ private static boolean containsFalse(Iterable<RexNode> nodes) {
+ for (RexNode node : nodes) {
+ if (node.isAlwaysFalse()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean containsTrue(Iterable<RexNode> nodes) {
+ for (RexNode node : nodes) {
+ if (node.isAlwaysTrue()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns a function that applies NOT to its argument.
+ *
+ * @deprecated Use {@link #not}
+ */
+ @SuppressWarnings("Guava")
+ @Deprecated // to be removed before 2.0
+ public static com.google.common.base.Function<RexNode, RexNode> notFn(
+ final RexBuilder rexBuilder) {
+ return e -> not(rexBuilder, e);
+ }
+
+ /**
+ * Applies NOT to an expression.
+ *
+ * <p>Unlike {@link #not}, may strengthen the type from {@code BOOLEAN} to {@code BOOLEAN NOT
+ * NULL}.
+ */
+ static RexNode not(final RexBuilder rexBuilder, RexNode input) {
+ return input.isAlwaysTrue()
+ ? rexBuilder.makeLiteral(false)
+ : input.isAlwaysFalse()
+ ? rexBuilder.makeLiteral(true)
+ : input.getKind() == SqlKind.NOT
+ ? ((RexCall) input).operands.get(0)
+ : rexBuilder.makeCall(SqlStdOperatorTable.NOT, input);
+ }
+
+ /** Returns whether an expression contains a {@link RexCorrelVariable}. */
+ public static boolean containsCorrelation(RexNode condition) {
+ try {
+ condition.accept(CorrelationFinder.INSTANCE);
+ return false;
+ } catch (Util.FoundOne e) {
+ return true;
+ }
+ }
+
+ /**
+ * Given an expression, it will swap the table references contained in its {@link
+ * RexTableInputRef} using the contents in the map.
+ */
+ public static RexNode swapTableReferences(
+ final RexBuilder rexBuilder,
+ final RexNode node,
+ final Map<RelTableRef, RelTableRef> tableMapping) {
+ return swapTableColumnReferences(rexBuilder, node, tableMapping, null);
+ }
+
+ /**
+ * Given an expression, it will swap its column references {@link RexTableInputRef} using the
+ * contents in the map (in particular, the first element of the set in the map value).
+ */
+ public static RexNode swapColumnReferences(
+ final RexBuilder rexBuilder,
+ final RexNode node,
+ final Map<RexTableInputRef, Set<RexTableInputRef>> ec) {
+ return swapTableColumnReferences(rexBuilder, node, null, ec);
+ }
+
+ /**
+ * Given an expression, it will swap the table references contained in its {@link
+ * RexTableInputRef} using the contents in the first map, and then it will swap the column
+ * references {@link RexTableInputRef} using the contents in the second map (in particular, the
+ * first element of the set in the map value).
+ */
+ public static RexNode swapTableColumnReferences(
+ final RexBuilder rexBuilder,
+ final RexNode node,
+ final @Nullable Map<RelTableRef, RelTableRef> tableMapping,
+ final @Nullable Map<RexTableInputRef, Set<RexTableInputRef>> ec) {
+ RexShuttle visitor =
+ new RexShuttle() {
+ @Override
+ public RexNode visitTableInputRef(RexTableInputRef inputRef) {
+ if (tableMapping != null) {
+ RexTableInputRef inputRefFinal = inputRef;
+ inputRef =
+ RexTableInputRef.of(
+ requireNonNull(
+ tableMapping.get(inputRef.getTableRef()),
+ () ->
+ "tableMapping.get(...) for "
+ + inputRefFinal.getTableRef()),
+ inputRef.getIndex(),
+ inputRef.getType());
+ }
+ if (ec != null) {
+ Set<RexTableInputRef> s = ec.get(inputRef);
+ if (s != null) {
+ inputRef = s.iterator().next();
+ }
+ }
+ return inputRef;
+ }
+ };
+ return visitor.apply(node);
+ }
+
+ /**
+ * Given an expression, it will swap the column references {@link RexTableInputRef} using the
+ * contents in the first map (in particular, the first element of the set in the map value), and
+ * then it will swap the table references contained in its {@link RexTableInputRef} using the
+ * contents in the second map.
+ */
+ public static RexNode swapColumnTableReferences(
+ final RexBuilder rexBuilder,
+ final RexNode node,
+ final Map<RexTableInputRef, ? extends @Nullable Set<RexTableInputRef>> ec,
+ final @Nullable Map<RelTableRef, RelTableRef> tableMapping) {
+ RexShuttle visitor =
+ new RexShuttle() {
+ @Override
+ public RexNode visitTableInputRef(RexTableInputRef inputRef) {
+ if (ec != null) {
+ Set<RexTableInputRef> s = ec.get(inputRef);
+ if (s != null) {
+ inputRef = s.iterator().next();
+ }
+ }
+ if (tableMapping != null) {
+ RexTableInputRef inputRefFinal = inputRef;
+ inputRef =
+ RexTableInputRef.of(
+ requireNonNull(
+ tableMapping.get(inputRef.getTableRef()),
+ () ->
+ "tableMapping.get(...) for "
+ + inputRefFinal.getTableRef()),
+ inputRef.getIndex(),
+ inputRef.getType());
+ }
+ return inputRef;
+ }
+ };
+ return visitor.apply(node);
+ }
+
+ /**
+ * Gather all table references in input expressions.
+ *
+ * @param nodes expressions
+ * @return set of table references
+ */
+ public static Set<RelTableRef> gatherTableReferences(final List<RexNode> nodes) {
+ final Set<RelTableRef> occurrences = new HashSet<>();
+ new RexVisitorImpl<Void>(true) {
+ @Override
+ public Void visitTableInputRef(RexTableInputRef ref) {
+ occurrences.add(ref.getTableRef());
+ return super.visitTableInputRef(ref);
+ }
+ }.visitEach(nodes);
+ return occurrences;
+ }
+
+ /** Given some expressions, gets the indices of the non-constant ones. */
+ public static ImmutableBitSet getNonConstColumns(List<RexNode> expressions) {
+ ImmutableBitSet cols = ImmutableBitSet.range(0, expressions.size());
+ return getNonConstColumns(cols, expressions);
+ }
+
+ /** Given some expressions and columns, gets the indices of the non-constant ones. */
+ public static ImmutableBitSet getNonConstColumns(
+ ImmutableBitSet columns, List<RexNode> expressions) {
+ ImmutableBitSet.Builder nonConstCols = ImmutableBitSet.builder();
+ for (int col : columns) {
+ if (!isLiteral(expressions.get(col), true)) {
+ nonConstCols.set(col);
+ }
+ }
+ return nonConstCols.build();
+ }
+
+ // ~ Inner Classes ----------------------------------------------------------
+
+ /** Walks over expressions and builds a bank of common sub-expressions. */
+ private static class ExpressionNormalizer extends RexVisitorImpl<@Nullable RexNode> {
+ final Map<RexNode, RexNode> map = new HashMap<>();
+ final boolean allowDups;
+
+ protected ExpressionNormalizer(boolean allowDups) {
+ super(true);
+ this.allowDups = allowDups;
+ }
+
+ protected RexNode register(RexNode expr) {
+ final RexNode previous = map.put(expr, expr);
+ if (!allowDups && (previous != null)) {
+ throw new SubExprExistsException(expr);
+ }
+ return expr;
+ }
+
+ protected RexNode lookup(RexNode expr) {
+ return requireNonNull(
+ map.get(expr), () -> "missing normalization for expression " + expr);
+ }
+
+ @Override
+ public RexNode visitInputRef(RexInputRef inputRef) {
+ return register(inputRef);
+ }
+
+ @Override
+ public RexNode visitLiteral(RexLiteral literal) {
+ return register(literal);
+ }
+
+ @Override
+ public RexNode visitCorrelVariable(RexCorrelVariable correlVariable) {
+ return register(correlVariable);
+ }
+
+ @Override
+ public RexNode visitCall(RexCall call) {
+ List<RexNode> normalizedOperands = new ArrayList<>();
+ int diffCount = 0;
+ for (RexNode operand : call.getOperands()) {
+ operand.accept(this);
+ final RexNode normalizedOperand = lookup(operand);
+ normalizedOperands.add(normalizedOperand);
+ if (normalizedOperand != operand) {
+ ++diffCount;
+ }
+ }
+ if (diffCount > 0) {
+ call = call.clone(call.getType(), normalizedOperands);
+ }
+ return register(call);
+ }
+
+ @Override
+ public RexNode visitDynamicParam(RexDynamicParam dynamicParam) {
+ return register(dynamicParam);
+ }
+
+ @Override
+ public RexNode visitRangeRef(RexRangeRef rangeRef) {
+ return register(rangeRef);
+ }
+
+ @Override
+ public RexNode visitFieldAccess(RexFieldAccess fieldAccess) {
+ final RexNode expr = fieldAccess.getReferenceExpr();
+ expr.accept(this);
+ final RexNode normalizedExpr = lookup(expr);
+ if (normalizedExpr != expr) {
+ fieldAccess = new RexFieldAccess(normalizedExpr, fieldAccess.getField());
+ }
+ return register(fieldAccess);
+ }
+
+ /** Thrown if there is a sub-expression. */
+ private static class SubExprExistsException extends ControlFlowException {
+ SubExprExistsException(RexNode expr) {
+ Util.discard(expr);
+ }
+ }
+ }
+
+ /**
+ * Walks over an expression and throws an exception if it finds an {@link RexInputRef} with an
+ * ordinal beyond the number of fields in the input row type, or a {@link RexLocalRef} with
+ * ordinal greater than that set using {@link #setLimit(int)}.
+ */
+ private static class ForwardRefFinder extends RexVisitorImpl<Void> {
+ private int limit = -1;
+ private final RelDataType inputRowType;
+
+ ForwardRefFinder(RelDataType inputRowType) {
+ super(true);
+ this.inputRowType = inputRowType;
+ }
+
+ @Override
+ public Void visitInputRef(RexInputRef inputRef) {
+ super.visitInputRef(inputRef);
+ if (inputRef.getIndex() >= inputRowType.getFieldCount()) {
+ throw new IllegalForwardRefException();
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLocalRef(RexLocalRef inputRef) {
+ super.visitLocalRef(inputRef);
+ if (inputRef.getIndex() >= limit) {
+ throw new IllegalForwardRefException();
+ }
+ return null;
+ }
+
+ public void setLimit(int limit) {
+ this.limit = limit;
+ }
+
+ /**
+ * Thrown to abort a visit when we find an illegal forward reference. It changes control
+ * flow but is not considered an error.
+ */
+ static class IllegalForwardRefException extends ControlFlowException {}
+ }
+
+ /** Visitor which builds a bitmap of the inputs used by an expression. */
+ public static class FieldAccessFinder extends RexVisitorImpl<Void> {
+ private final List<RexFieldAccess> fieldAccessList;
+
+ public FieldAccessFinder() {
+ super(true);
+ fieldAccessList = new ArrayList<>();
+ }
+
+ @Override
+ public Void visitFieldAccess(RexFieldAccess fieldAccess) {
+ fieldAccessList.add(fieldAccess);
+ return null;
+ }
+
+ @Override
+ public Void visitCall(RexCall call) {
+ visitEach(call.operands);
+ return null;
+ }
+
+ public List<RexFieldAccess> getFieldAccessList() {
+ return fieldAccessList;
+ }
+ }
+
+ /** Helps {@link org.apache.calcite.rex.RexUtil#toCnf}. */
+ private static class CnfHelper {
+ final RexBuilder rexBuilder;
+ int currentCount;
+ final int maxNodeCount; // negative means no limit
+
+ private CnfHelper(RexBuilder rexBuilder, int maxNodeCount) {
+ this.rexBuilder = rexBuilder;
+ this.maxNodeCount = maxNodeCount;
+ }
+
+ public RexNode toCnf(RexNode rex) {
+ try {
+ this.currentCount = 0;
+ return toCnf2(rex);
+ } catch (OverflowError e) {
+ Util.swallow(e, null);
+ return rex;
+ }
+ }
+
+ private RexNode toCnf2(RexNode rex) {
+ final List<RexNode> operands;
+ switch (rex.getKind()) {
+ case AND:
+ incrementAndCheck();
+ operands = flattenAnd(((RexCall) rex).getOperands());
+ final List<RexNode> cnfOperands = new ArrayList<>();
+ for (RexNode node : operands) {
+ RexNode cnf = toCnf2(node);
+ switch (cnf.getKind()) {
+ case AND:
+ incrementAndCheck();
+ cnfOperands.addAll(((RexCall) cnf).getOperands());
+ break;
+ default:
+ incrementAndCheck();
+ cnfOperands.add(cnf);
+ }
+ }
+ return and(cnfOperands);
+ case OR:
+ incrementAndCheck();
+ operands = flattenOr(((RexCall) rex).getOperands());
+ final RexNode head = operands.get(0);
+ final RexNode headCnf = toCnf2(head);
+ final List<RexNode> headCnfs = RelOptUtil.conjunctions(headCnf);
+ final RexNode tail = or(Util.skip(operands));
+ final RexNode tailCnf = toCnf2(tail);
+ final List<RexNode> tailCnfs = RelOptUtil.conjunctions(tailCnf);
+ final List<RexNode> list = new ArrayList<>();
+ for (RexNode h : headCnfs) {
+ for (RexNode t : tailCnfs) {
+ list.add(or(ImmutableList.of(h, t)));
+ }
+ }
+ return and(list);
+ case NOT:
+ final RexNode arg = ((RexCall) rex).getOperands().get(0);
+ switch (arg.getKind()) {
+ case NOT:
+ return toCnf2(((RexCall) arg).getOperands().get(0));
+ case OR:
+ operands = ((RexCall) arg).getOperands();
+ return toCnf2(
+ and(Util.transform(flattenOr(operands), RexUtil::addNot)));
+ case AND:
+ operands = ((RexCall) arg).getOperands();
+ return toCnf2(
+ or(Util.transform(flattenAnd(operands), RexUtil::addNot)));
+ default:
+ incrementAndCheck();
+ return rex;
+ }
+ default:
+ incrementAndCheck();
+ return rex;
+ }
+ }
+
+ private void incrementAndCheck() {
+ if (maxNodeCount >= 0 && ++currentCount > maxNodeCount) {
+ throw OverflowError.INSTANCE;
+ }
+ }
+
+ /** Exception to catch when we pass the limit. */
+ @SuppressWarnings("serial")
+ private static class OverflowError extends ControlFlowException {
+ @SuppressWarnings("ThrowableInstanceNeverThrown")
+ protected static final OverflowError INSTANCE = new OverflowError();
+
+ private OverflowError() {}
+ }
+
+ private RexNode pull(RexNode rex) {
+ final List<RexNode> operands;
+ switch (rex.getKind()) {
+ case AND:
+ operands = flattenAnd(((RexCall) rex).getOperands());
+ return and(pullList(operands));
+ case OR:
+ operands = flattenOr(((RexCall) rex).getOperands());
+ final Map<RexNode, RexNode> factors = commonFactors(operands);
+ if (factors.isEmpty()) {
+ return or(operands);
+ }
+ final List<RexNode> list = new ArrayList<>();
+ for (RexNode operand : operands) {
+ list.add(removeFactor(factors, operand));
+ }
+ return and(Iterables.concat(factors.values(), ImmutableList.of(or(list))));
+ default:
+ return rex;
+ }
+ }
+
+ private List<RexNode> pullList(List<RexNode> nodes) {
+ final List<RexNode> list = new ArrayList<>();
+ for (RexNode node : nodes) {
+ RexNode pulled = pull(node);
+ switch (pulled.getKind()) {
+ case AND:
+ list.addAll(((RexCall) pulled).getOperands());
+ break;
+ default:
+ list.add(pulled);
+ }
+ }
+ return list;
+ }
+
+ private static Map<RexNode, RexNode> commonFactors(List<RexNode> nodes) {
+ final Map<RexNode, RexNode> map = new HashMap<>();
+ int i = 0;
+ for (RexNode node : nodes) {
+ if (i++ == 0) {
+ for (RexNode conjunction : RelOptUtil.conjunctions(node)) {
+ map.put(conjunction, conjunction);
+ }
+ } else {
+ map.keySet().retainAll(RelOptUtil.conjunctions(node));
+ }
+ }
+ return map;
+ }
+
+ private RexNode removeFactor(Map<RexNode, RexNode> factors, RexNode node) {
+ List<RexNode> list = new ArrayList<>();
+ for (RexNode operand : RelOptUtil.conjunctions(node)) {
+ if (!factors.containsKey(operand)) {
+ list.add(operand);
+ }
+ }
+ return and(list);
+ }
+
+ private RexNode and(Iterable<? extends RexNode> nodes) {
+ return composeConjunction(rexBuilder, nodes);
+ }
+
+ private RexNode or(Iterable<? extends RexNode> nodes) {
+ return composeDisjunction(rexBuilder, nodes);
+ }
+ }
+
+ /** Transforms a list of expressions to the list of digests. */
+ public static List<String> strings(List<RexNode> list) {
+ return Util.transform(list, Object::toString);
+ }
+
+ /** Helps {@link org.apache.calcite.rex.RexUtil#toDnf}. */
+ private static class DnfHelper {
+ final RexBuilder rexBuilder;
+
+ private DnfHelper(RexBuilder rexBuilder) {
+ this.rexBuilder = rexBuilder;
+ }
+
+ public RexNode toDnf(RexNode rex) {
+ final List<RexNode> operands;
+ switch (rex.getKind()) {
+ case AND:
+ operands = flattenAnd(((RexCall) rex).getOperands());
+ final RexNode head = operands.get(0);
+ final RexNode headDnf = toDnf(head);
+ final List<RexNode> headDnfs = RelOptUtil.disjunctions(headDnf);
+ final RexNode tail = and(Util.skip(operands));
+ final RexNode tailDnf = toDnf(tail);
+ final List<RexNode> tailDnfs = RelOptUtil.disjunctions(tailDnf);
+ final List<RexNode> list = new ArrayList<>();
+ for (RexNode h : headDnfs) {
+ for (RexNode t : tailDnfs) {
+ list.add(and(ImmutableList.of(h, t)));
+ }
+ }
+ return or(list);
+ case OR:
+ operands = flattenOr(((RexCall) rex).getOperands());
+ return or(toDnfs(operands));
+ case NOT:
+ final RexNode arg = ((RexCall) rex).getOperands().get(0);
+ switch (arg.getKind()) {
+ case NOT:
+ return toDnf(((RexCall) arg).getOperands().get(0));
+ case OR:
+ operands = ((RexCall) arg).getOperands();
+ return toDnf(and(Util.transform(flattenOr(operands), RexUtil::addNot)));
+ case AND:
+ operands = ((RexCall) arg).getOperands();
+ return toDnf(or(Util.transform(flattenAnd(operands), RexUtil::addNot)));
+ default:
+ return rex;
+ }
+ default:
+ return rex;
+ }
+ }
+
+ private List<RexNode> toDnfs(List<RexNode> nodes) {
+ final List<RexNode> list = new ArrayList<>();
+ for (RexNode node : nodes) {
+ RexNode dnf = toDnf(node);
+ switch (dnf.getKind()) {
+ case OR:
+ list.addAll(((RexCall) dnf).getOperands());
+ break;
+ default:
+ list.add(dnf);
+ }
+ }
+ return list;
+ }
+
+ private RexNode and(Iterable<? extends RexNode> nodes) {
+ return composeConjunction(rexBuilder, nodes);
+ }
+
+ private RexNode or(Iterable<? extends RexNode> nodes) {
+ return composeDisjunction(rexBuilder, nodes);
+ }
+ }
+
+ /** Shuttle that adds {@code offset} to each {@link RexInputRef} in an expression. */
+ private static class RexShiftShuttle extends RexShuttle {
+ private final int offset;
+
+ RexShiftShuttle(int offset) {
+ this.offset = offset;
+ }
+
+ @Override
+ public RexNode visitInputRef(RexInputRef input) {
+ return new RexInputRef(input.getIndex() + offset, input.getType());
+ }
+ }
+
+ /**
+ * Visitor that throws {@link org.apache.calcite.util.Util.FoundOne} if applied to an expression
+ * that contains a {@link RexCorrelVariable}.
+ */
+ private static class CorrelationFinder extends RexVisitorImpl<Void> {
+ static final CorrelationFinder INSTANCE = new CorrelationFinder();
+
+ private CorrelationFinder() {
+ super(true);
+ }
+
+ @Override
+ public Void visitCorrelVariable(RexCorrelVariable var) {
+ throw Util.FoundOne.NULL;
+ }
+ }
+
+ /** Shuttle that fixes up an expression to match changes in nullability of input fields. */
+ public static class FixNullabilityShuttle extends RexShuttle {
+ private final List<RelDataType> typeList;
+ private final RexBuilder rexBuilder;
+
+ public FixNullabilityShuttle(RexBuilder rexBuilder, List<RelDataType> typeList) {
+ this.typeList = typeList;
+ this.rexBuilder = rexBuilder;
+ }
+
+ @Override
+ public RexNode visitInputRef(RexInputRef ref) {
+ final RelDataType rightType = typeList.get(ref.getIndex());
+ final RelDataType refType = ref.getType();
+ if (refType.equals(rightType)) {
+ return ref;
+ }
+ final RelDataType refType2 =
+ rexBuilder
+ .getTypeFactory()
+ .createTypeWithNullability(refType, rightType.isNullable());
+ if (refType2.equals(rightType)) {
+ return new RexInputRef(ref.getIndex(), refType2);
+ }
+ throw new AssertionError("mismatched type " + ref + " " + rightType);
+ }
+ }
+
+ /**
+ * Visitor that throws {@link org.apache.calcite.util.Util.FoundOne} if applied to an expression
+ * that contains a {@link RexSubQuery}.
+ */
+ public static class SubQueryFinder extends RexVisitorImpl<Void> {
+ public static final SubQueryFinder INSTANCE = new SubQueryFinder();
+
+ @SuppressWarnings("Guava")
+ @Deprecated // to be removed before 2.0
+ public static final com.google.common.base.Predicate<Project> PROJECT_PREDICATE =
+ SubQueryFinder::containsSubQuery;
+
+ @SuppressWarnings("Guava")
+ @Deprecated // to be removed before 2.0
+ public static final Predicate<Filter> FILTER_PREDICATE = SubQueryFinder::containsSubQuery;
+
+ @SuppressWarnings("Guava")
+ @Deprecated // to be removed before 2.0
+ public static final com.google.common.base.Predicate<Join> JOIN_PREDICATE =
+ SubQueryFinder::containsSubQuery;
+
+ private SubQueryFinder() {
+ super(true);
+ }
+
+ /** Returns whether a {@link Project} contains a sub-query. */
+ public static boolean containsSubQuery(Project project) {
+ for (RexNode node : project.getProjects()) {
+ try {
+ node.accept(INSTANCE);
+ } catch (Util.FoundOne e) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** Returns whether a {@link Filter} contains a sub-query. */
+ public static boolean containsSubQuery(Filter filter) {
+ try {
+ filter.getCondition().accept(INSTANCE);
+ return false;
+ } catch (Util.FoundOne e) {
+ return true;
+ }
+ }
+
+ /** Returns whether a {@link Join} contains a sub-query. */
+ public static boolean containsSubQuery(Join join) {
+ try {
+ join.getCondition().accept(INSTANCE);
+ return false;
+ } catch (Util.FoundOne e) {
+ return true;
+ }
+ }
+
+ @Override
+ public Void visitSubQuery(RexSubQuery subQuery) {
+ throw new Util.FoundOne(subQuery);
+ }
+
+ public static @Nullable RexSubQuery find(Iterable<RexNode> nodes) {
+ for (RexNode node : nodes) {
+ try {
+ node.accept(INSTANCE);
+ } catch (Util.FoundOne e) {
+ return (RexSubQuery) e.getNode();
+ }
+ }
+ return null;
+ }
+
+ public static @Nullable RexSubQuery find(RexNode node) {
+ try {
+ node.accept(INSTANCE);
+ return null;
+ } catch (Util.FoundOne e) {
+ return (RexSubQuery) e.getNode();
+ }
+ }
+ }
+
+ /**
+ * Deep expressions simplifier.
+ *
+ * <p>This class is broken because it does not change the value of {@link RexUnknownAs} as it
+ * recurses into an expression. Do not use.
+ */
+ @Deprecated // to be removed before 2.0
+ public static class ExprSimplifier extends RexShuttle {
+ private final RexSimplify simplify;
+ private final Map<RexNode, RexUnknownAs> unknownAsMap = new HashMap<>();
+ private final RexUnknownAs unknownAs;
+ private final boolean matchNullability;
+
+ public ExprSimplifier(RexSimplify simplify) {
+ this(simplify, RexUnknownAs.UNKNOWN, true);
+ }
+
+ public ExprSimplifier(RexSimplify simplify, boolean matchNullability) {
+ this(simplify, RexUnknownAs.UNKNOWN, matchNullability);
+ }
+
+ public ExprSimplifier(
+ RexSimplify simplify, RexUnknownAs unknownAs, boolean matchNullability) {
+ this.simplify = simplify;
+ this.unknownAs = unknownAs;
+ this.matchNullability = matchNullability;
+ }
+
+ @Override
+ public RexNode visitCall(RexCall call) {
+ RexUnknownAs unknownAs = this.unknownAs;
+ switch (unknownAs) {
+ case FALSE:
+ switch (call.getKind()) {
+ case AND:
+ case CASE:
+ // Default value is used for top operator
+ unknownAs = unknownAsMap.getOrDefault(call, RexUnknownAs.FALSE);
+ break;
+ default:
+ unknownAs = RexUnknownAs.FALSE;
+ }
+ for (RexNode operand : call.operands) {
+ this.unknownAsMap.put(operand, unknownAs);
+ }
+ break;
+ default:
+ break;
+ }
+ RexNode node = super.visitCall(call);
+ RexNode simplifiedNode = simplify.simplify(node, unknownAs);
+ if (node == simplifiedNode) {
+ return node;
+ }
+ if (simplifiedNode.getType().equals(call.getType())) {
+ return simplifiedNode;
+ }
+ return simplify.rexBuilder.makeCast(call.getType(), simplifiedNode, matchNullability);
+ }
+ }
+
+ /** Visitor that tells whether a node matching a particular description exists in a tree. */
+ public abstract static class RexFinder extends RexVisitorImpl<Void> {
+ RexFinder() {
+ super(true);
+ }
+
+ /** Returns whether a {@link Project} contains the kind of expression we seek. */
+ public boolean inProject(Project project) {
+ return anyContain(project.getProjects());
+ }
+
+ /** Returns whether a {@link Filter} contains the kind of expression we seek. */
+ public boolean inFilter(Filter filter) {
+ return contains(filter.getCondition());
+ }
+
+ /** Returns whether a {@link Join} contains kind of expression we seek. */
+ public boolean inJoin(Join join) {
+ return contains(join.getCondition());
+ }
+
+ /** Returns whether the given expression contains what this RexFinder seeks. */
+ public boolean contains(RexNode node) {
+ try {
+ node.accept(RexFinder.this);
+ return false;
+ } catch (Util.FoundOne e) {
+ return true;
+ }
+ }
+
+ /** Returns whether any of the given expressions contain what this RexFinder seeks. */
+ public boolean anyContain(Iterable<? extends RexNode> nodes) {
+ try {
+ for (RexNode node : nodes) {
+ node.accept(RexFinder.this);
+ }
+ return false;
+ } catch (Util.FoundOne e) {
+ return true;
+ }
+ }
+ }
+
+ /**
+ * Converts a {@link Range} to a {@link RexNode} expression.
+ *
+ * @param <C> Value type
+ */
+ private static class RangeToRex<C extends Comparable<C>> implements RangeSets.Consumer<C> {
+ private final List<RexNode> list;
+ private final RexBuilder rexBuilder;
+ private final RelDataType type;
+ private final RexNode ref;
+
+ RangeToRex(RexNode ref, List<RexNode> list, RexBuilder rexBuilder, RelDataType type) {
+ this.ref = requireNonNull(ref, "ref");
+ this.list = requireNonNull(list, "list");
+ this.rexBuilder = requireNonNull(rexBuilder, "rexBuilder");
+ this.type = requireNonNull(type, "type");
+ }
+
+ private void addAnd(RexNode... nodes) {
+ list.add(rexBuilder.makeCall(SqlStdOperatorTable.AND, nodes));
+ }
+
+ private RexNode op(SqlOperator op, C value) {
+ return rexBuilder.makeCall(op, ref, rexBuilder.makeLiteral(value, type, true, true));
+ }
+
+ @Override
+ public void all() {
+ list.add(rexBuilder.makeLiteral(true));
+ }
+
+ @Override
+ public void atLeast(C lower) {
+ list.add(op(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, lower));
+ }
+
+ @Override
+ public void atMost(C upper) {
+ list.add(op(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, upper));
+ }
+
+ @Override
+ public void greaterThan(C lower) {
+ list.add(op(SqlStdOperatorTable.GREATER_THAN, lower));
+ }
+
+ @Override
+ public void lessThan(C upper) {
+ list.add(op(SqlStdOperatorTable.LESS_THAN, upper));
+ }
+
+ @Override
+ public void singleton(C value) {
+ list.add(op(SqlStdOperatorTable.EQUALS, value));
+ }
+
+ @Override
+ public void closed(C lower, C upper) {
+ addAnd(
+ op(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, lower),
+ op(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, upper));
+ }
+
+ @Override
+ public void closedOpen(C lower, C upper) {
+ addAnd(
+ op(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, lower),
+ op(SqlStdOperatorTable.LESS_THAN, upper));
+ }
+
+ @Override
+ public void openClosed(C lower, C upper) {
+ addAnd(
+ op(SqlStdOperatorTable.GREATER_THAN, lower),
+ op(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, upper));
+ }
+
+ @Override
+ public void open(C lower, C upper) {
+ addAnd(
+ op(SqlStdOperatorTable.GREATER_THAN, lower),
+ op(SqlStdOperatorTable.LESS_THAN, upper));
+ }
+ }
+
+ /**
+ * Shuttle that expands calls to {@link org.apache.calcite.sql.fun.SqlStdOperatorTable#SEARCH}.
+ *
+ * <p>Calls whose complexity is greater than {@link #maxComplexity} are retained (not expanded).
+ */
+ private static class SearchExpandingShuttle extends RexShuttle {
+ private final RexBuilder rexBuilder;
+ private final @Nullable RexProgram program;
+ private final int maxComplexity;
+
+ SearchExpandingShuttle(
+ @Nullable RexProgram program, RexBuilder rexBuilder, int maxComplexity) {
+ this.program = program;
+ this.rexBuilder = rexBuilder;
+ this.maxComplexity = maxComplexity;
+ }
+
+ @Override
+ public RexNode visitCall(RexCall call) {
+ final boolean[] update = {false};
+ final List<RexNode> clonedOperands;
+ switch (call.getKind()) {
+ // Flatten AND/OR operands.
+ case OR:
+ clonedOperands = visitList(call.operands, update);
+ if (update[0]) {
+ return composeDisjunction(rexBuilder, clonedOperands);
+ } else {
+ return call;
+ }
+ case AND:
+ clonedOperands = visitList(call.operands, update);
+ if (update[0]) {
+ return composeConjunction(rexBuilder, clonedOperands);
+ } else {
+ return call;
+ }
+ case SEARCH:
+ final RexNode ref = call.operands.get(0);
+ final RexLiteral literal = (RexLiteral) deref(program, call.operands.get(1));
+ final Sarg sarg = requireNonNull(literal.getValueAs(Sarg.class), "Sarg");
+ if (maxComplexity < 0 || sarg.complexity() < maxComplexity) {
+ return sargRef(
+ rexBuilder, ref, sarg, literal.getType(), RexUnknownAs.UNKNOWN);
+ }
+ // Sarg is complex (therefore useful); fall through
+ default:
+ return super.visitCall(call);
+ }
+ }
+ }
+}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlTimestampAddFunction.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlTimestampAddFunction.java
index da04a18ad59..109180bc38e 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlTimestampAddFunction.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlTimestampAddFunction.java
@@ -19,13 +19,21 @@ package org.apache.calcite.sql.fun;
import org.apache.calcite.avatica.util.TimeUnit;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlIntervalQualifier;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.type.OperandTypes;
import org.apache.calcite.sql.type.SqlReturnTypeInference;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.type.SqlTypeTransforms;
+import org.apache.calcite.sql.validate.SqlValidator;
+import org.apache.calcite.sql.validate.SqlValidatorScope;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+import static org.apache.calcite.util.Util.first;
/**
* The <code>TIMESTAMPADD</code> function, which adds an interval to a datetime (TIMESTAMP, TIME or
@@ -66,14 +74,11 @@ public class SqlTimestampAddFunction extends SqlFunction {
private static final int MICROSECOND_PRECISION = 6;
private static final SqlReturnTypeInference RETURN_TYPE_INFERENCE =
- opBinding -> {
- final RelDataTypeFactory typeFactory = opBinding.getTypeFactory();
- return deduceType(
- typeFactory,
- opBinding.getOperandLiteralValue(0, TimeUnit.class),
- opBinding.getOperandType(1),
- opBinding.getOperandType(2));
- };
+ opBinding ->
+ deduceType(
+ opBinding.getTypeFactory(),
+ opBinding.getOperandLiteralValue(0, TimeUnit.class),
+ opBinding.getOperandType(2));
// BEGIN FLINK MODIFICATION
// Reason: this method is changed to deduce return type on timestamp with local time zone
@@ -81,57 +86,67 @@ public class SqlTimestampAddFunction extends SqlFunction {
// Whole class should be removed after CALCITE-4698 is fixed
public static RelDataType deduceType(
RelDataTypeFactory typeFactory,
- TimeUnit timeUnit,
- RelDataType intervalType,
- RelDataType datetimeType) {
- RelDataType type;
- switch (timeUnit) {
+ @Nullable TimeUnit timeUnit,
+ RelDataType operandType1,
+ RelDataType operandType2) {
+ final RelDataType type = deduceType(typeFactory, timeUnit, operandType2);
+ return typeFactory.createTypeWithNullability(
+ type, operandType1.isNullable() || operandType2.isNullable());
+ }
+
+ static RelDataType deduceType(
+ RelDataTypeFactory typeFactory, @Nullable TimeUnit timeUnit, RelDataType datetimeType) {
+ final TimeUnit timeUnit2 = first(timeUnit, TimeUnit.EPOCH);
+ SqlTypeName typeName = datetimeType.getSqlTypeName();
+ switch (timeUnit2) {
case MILLISECOND:
- type =
- typeFactory.createSqlType(
- timestampOrTimestampLtz(datetimeType),
- Math.max(MILLISECOND_PRECISION, datetimeType.getPrecision()));
- break;
+ return typeFactory.createSqlType(
+ typeName, Math.max(MILLISECOND_PRECISION, datetimeType.getPrecision()));
case MICROSECOND:
- type =
- typeFactory.createSqlType(
- timestampOrTimestampLtz(datetimeType),
- Math.max(MICROSECOND_PRECISION, datetimeType.getPrecision()));
- break;
+ return typeFactory.createSqlType(
+ typeName, Math.max(MICROSECOND_PRECISION, datetimeType.getPrecision()));
case HOUR:
case MINUTE:
case SECOND:
if (datetimeType.getFamily() == SqlTypeFamily.TIME) {
- type = datetimeType;
+ return datetimeType;
} else if (datetimeType.getFamily() == SqlTypeFamily.TIMESTAMP) {
- type =
- typeFactory.createSqlType(
- timestampOrTimestampLtz(datetimeType),
- datetimeType.getPrecision());
+ return typeFactory.createSqlType(typeName, datetimeType.getPrecision());
} else {
- type = typeFactory.createSqlType(SqlTypeName.TIMESTAMP);
+ return typeFactory.createSqlType(SqlTypeName.TIMESTAMP);
}
- break;
default:
- type = datetimeType;
+ return datetimeType;
}
- return typeFactory.createTypeWithNullability(
- type, intervalType.isNullable() || datetimeType.isNullable());
}
- private static SqlTypeName timestampOrTimestampLtz(RelDataType datetimeType) {
- return datetimeType.getSqlTypeName() == SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE
- ? SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE
- : SqlTypeName.TIMESTAMP;
+ @Override
+ public void validateCall(
+ SqlCall call,
+ SqlValidator validator,
+ SqlValidatorScope scope,
+ SqlValidatorScope operandScope) {
+ super.validateCall(call, validator, scope, operandScope);
+
+ // This is either a time unit or a time frame:
+ //
+ // * In "TIMESTAMPADD(YEAR, 2, x)" operand 0 is a SqlIntervalQualifier
+ // with startUnit = YEAR and timeFrameName = null.
+ //
+ // * In "TIMESTAMPADD(MINUTE15, 2, x) operand 0 is a SqlIntervalQualifier
+ // with startUnit = EPOCH and timeFrameName = 'MINUTE15'.
+ //
+ // If the latter, check that timeFrameName is valid.
+ validator.validateTimeFrame((SqlIntervalQualifier) call.getOperandList().get(0));
}
// END FLINK MODIFICATION
/** Creates a SqlTimestampAddFunction. */
- SqlTimestampAddFunction() {
+ SqlTimestampAddFunction(String name) {
super(
- "TIMESTAMPADD",
+ name,
SqlKind.TIMESTAMP_ADD,
- RETURN_TYPE_INFERENCE,
+ RETURN_TYPE_INFERENCE.andThen(SqlTypeTransforms.TO_NULLABLE),
null,
OperandTypes.family(
SqlTypeFamily.ANY, SqlTypeFamily.INTEGER, SqlTypeFamily.DATETIME),
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/type/CompositeSingleOperandTypeChecker.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/type/CompositeSingleOperandTypeChecker.java
new file mode 100644
index 00000000000..d7214ea8765
--- /dev/null
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/type/CompositeSingleOperandTypeChecker.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.sql.type;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.calcite.sql.SqlCallBinding;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.util.Util;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+/**
+ * Default implementation of {@link org.apache.calcite.sql.type.CompositeOperandTypeChecker}, the
+ * class was copied over because of current Calcite issue CALCITE-5380.
+ *
+ * <p>Lines 73 ~ 78, 101 ~ 107
+ */
+public class CompositeSingleOperandTypeChecker extends CompositeOperandTypeChecker
+ implements SqlSingleOperandTypeChecker {
+
+ // ~ Constructors -----------------------------------------------------------
+
+ /**
+ * Creates a CompositeSingleOperandTypeChecker. Outside this package, use {@link
+ * SqlSingleOperandTypeChecker#and(SqlSingleOperandTypeChecker)}, {@link OperandTypes#and},
+ * {@link OperandTypes#or} and similar.
+ */
+ CompositeSingleOperandTypeChecker(
+ CompositeOperandTypeChecker.Composition composition,
+ ImmutableList<? extends SqlSingleOperandTypeChecker> allowedRules,
+ @Nullable String allowedSignatures) {
+ super(composition, allowedRules, allowedSignatures, null, null);
+ }
+
+ // ~ Methods ----------------------------------------------------------------
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public ImmutableList<? extends SqlSingleOperandTypeChecker> getRules() {
+ return (ImmutableList<? extends SqlSingleOperandTypeChecker>) allowedRules;
+ }
+
+ @Override
+ public boolean checkSingleOperandType(
+ SqlCallBinding callBinding, SqlNode node, int iFormalOperand, boolean throwOnFailure) {
+ assert allowedRules.size() >= 1;
+
+ final ImmutableList<? extends SqlSingleOperandTypeChecker> rules = getRules();
+ if (composition == Composition.SEQUENCE) {
+ return rules.get(iFormalOperand)
+ .checkSingleOperandType(callBinding, node, 0, throwOnFailure);
+ }
+
+ int typeErrorCount = 0;
+
+ boolean throwOnAndFailure = (composition == Composition.AND) && throwOnFailure;
+
+ for (SqlSingleOperandTypeChecker rule : rules) {
+ if (!rule.checkSingleOperandType(
+ // FLINK MODIFICATION BEGIN
+ callBinding,
+ node,
+ rule.getClass() == FamilyOperandTypeChecker.class ? 0 : iFormalOperand,
+ throwOnAndFailure)) {
+ // FLINK MODIFICATION END
+ typeErrorCount++;
+ }
+ }
+
+ boolean ret;
+ switch (composition) {
+ case AND:
+ ret = typeErrorCount == 0;
+ break;
+ case OR:
+ ret = typeErrorCount < allowedRules.size();
+ break;
+ default:
+ // should never come here
+ throw Util.unexpected(composition);
+ }
+
+ if (!ret && throwOnFailure) {
+ // In the case of a composite OR, we want to throw an error
+ // describing in more detail what the problem was, hence doing the
+ // loop again.
+ for (SqlSingleOperandTypeChecker rule : rules) {
+ // FLINK MODIFICATION BEGIN
+ rule.checkSingleOperandType(
+ callBinding,
+ node,
+ rule.getClass() == FamilyOperandTypeChecker.class ? 0 : iFormalOperand,
+ true);
+ // FLINK MODIFICATION END
+ }
+
+ // If no exception thrown, just throw a generic validation signature
+ // error.
+ throw callBinding.newValidationSignatureError();
+ }
+
+ return ret;
+ }
+}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java
index eccac441204..a5f42aaeb13 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java
@@ -38,8 +38,9 @@ import static java.util.Objects.requireNonNull;
* <p>FLINK modifications are at lines
*
* <ol>
- * <li>Should be removed after fixing CALCITE-6342: Lines 475-485
- * <li>Should be removed after fix of FLINK-31350: Lines 552 ~ 564.
+ * <li>Should be removed after fixing CALCITE-6342: Lines 100-102
+ * <li>Should be removed after fixing CALCITE-6342: Lines 482-494
+ * <li>Should be removed after fix of FLINK-31350: Lines 561-573.
* </ol>
*/
public class SqlTypeFactoryImpl extends RelDataTypeFactoryImpl {
@@ -121,6 +122,12 @@ public class SqlTypeFactoryImpl extends RelDataTypeFactoryImpl {
return canonize(newType);
}
+ @Override
+ public RelDataType createMeasureType(RelDataType valueType) {
+ MeasureSqlType newType = MeasureSqlType.create(valueType);
+ return canonize(newType);
+ }
+
@Override
public RelDataType createSqlIntervalType(SqlIntervalQualifier intervalQualifier) {
RelDataType newType = new IntervalSqlType(typeSystem, intervalQualifier, false);
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
index 1d70c6890eb..097a9f2316f 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
@@ -29,11 +29,15 @@ import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.prepare.Prepare;
import org.apache.calcite.rel.type.DynamicRecordType;
+import org.apache.calcite.rel.type.RelCrossType;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rel.type.RelDataTypeSystem;
import org.apache.calcite.rel.type.RelRecordType;
+import org.apache.calcite.rel.type.TimeFrame;
+import org.apache.calcite.rel.type.TimeFrameSet;
+import org.apache.calcite.rel.type.TimeFrames;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexPatternFieldRef;
import org.apache.calcite.rex.RexVisitor;
@@ -79,6 +83,7 @@ import org.apache.calcite.sql.SqlSelectKeyword;
import org.apache.calcite.sql.SqlSnapshot;
import org.apache.calcite.sql.SqlSyntax;
import org.apache.calcite.sql.SqlTableFunction;
+import org.apache.calcite.sql.SqlUnknownLiteral;
import org.apache.calcite.sql.SqlUnpivot;
import org.apache.calcite.sql.SqlUnresolvedFunction;
import org.apache.calcite.sql.SqlUpdate;
@@ -160,19 +165,19 @@ import static org.apache.calcite.util.Static.RESOURCE;
* Default implementation of {@link SqlValidator}, the class was copied over because of
* CALCITE-4554.
*
- * <p>Lines 193 ~ 196, Flink improves error message for functions without appropriate arguments in
+ * <p>Lines 200 ~ 203, Flink improves error message for functions without appropriate arguments in
* handleUnresolvedFunction.
*
- * <p>Lines 1963 ~ 1983, Flink improves error message for functions without appropriate arguments in
+ * <p>Lines 2000 ~ 2020, Flink improves error message for functions without appropriate arguments in
* handleUnresolvedFunction at {@link SqlValidatorImpl#handleUnresolvedFunction}.
*
- * <p>Lines 3741 ~ 3745, Flink improves Optimize the retrieval of sub-operands in SqlCall when using
- * NamedParameters at {@link SqlValidatorImpl#checkRollUp}.
+ * <p>Lines 3814 ~ 3818, 6458 ~ 6464 Flink improves Optimize the retrieval of sub-operands in
+ * SqlCall when using NamedParameters at {@link SqlValidatorImpl#checkRollUp}.
*
- * <p>Lines 5113 ~ 5126, Flink enables TIMESTAMP and TIMESTAMP_LTZ for system time period
+ * <p>Lines 5196 ~ 5209, Flink enables TIMESTAMP and TIMESTAMP_LTZ for system time period
* specification type at {@link org.apache.calcite.sql.validate.SqlValidatorImpl#validateSnapshot}.
*
- * <p>Lines 5470 ~ 5476, Flink enables TIMESTAMP and TIMESTAMP_LTZ for first orderBy column in
+ * <p>Lines 5553 ~ 5559, Flink enables TIMESTAMP and TIMESTAMP_LTZ for first orderBy column in
* matchRecognize at {@link SqlValidatorImpl#validateMatchRecognize}.
*/
public class SqlValidatorImpl implements SqlValidatorWithHints {
@@ -242,12 +247,11 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
private int nextGeneratedId;
protected final RelDataTypeFactory typeFactory;
-
- /** The type of dynamic parameters until a type is imposed on them. */
protected final RelDataType unknownType;
-
private final RelDataType booleanType;
+ protected final TimeFrameSet timeFrameSet;
+
/**
* Map of derived RelDataType for each node. This is an IdentityHashMap since in some cases
* (such as null literals) we need to discriminate by instance.
@@ -300,9 +304,13 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
this.opTab = requireNonNull(opTab, "opTab");
this.catalogReader = requireNonNull(catalogReader, "catalogReader");
this.typeFactory = requireNonNull(typeFactory, "typeFactory");
+ final RelDataTypeSystem typeSystem = typeFactory.getTypeSystem();
+ this.timeFrameSet =
+ requireNonNull(typeSystem.deriveTimeFrameSet(TimeFrames.CORE), "timeFrameSet");
this.config = requireNonNull(config, "config");
- unknownType = typeFactory.createUnknownType();
+ // It is assumed that unknown type is nullable by default
+ unknownType = typeFactory.createTypeWithNullability(typeFactory.createUnknownType(), true);
booleanType = typeFactory.createSqlType(SqlTypeName.BOOLEAN);
final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
@@ -314,7 +322,30 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
@SuppressWarnings("argument.type.incompatible")
TypeCoercion typeCoercion = config.typeCoercionFactory().create(typeFactory, this);
this.typeCoercion = typeCoercion;
- if (config.typeCoercionRules() != null) {
+
+ if (config.conformance().allowCoercionStringToArray()) {
+ SqlTypeCoercionRule rules =
+ requireNonNull(
+ config.typeCoercionRules() != null
+ ? config.typeCoercionRules()
+ : SqlTypeCoercionRule.THREAD_PROVIDERS.get());
+
+ ImmutableSet<SqlTypeName> arrayMapping =
+ ImmutableSet.<SqlTypeName>builder()
+ .addAll(
+ rules.getTypeMapping()
+ .getOrDefault(SqlTypeName.ARRAY, ImmutableSet.of()))
+ .add(SqlTypeName.VARCHAR)
+ .add(SqlTypeName.CHAR)
+ .build();
+
+ Map<SqlTypeName, ImmutableSet<SqlTypeName>> mapping =
+ new HashMap(rules.getTypeMapping());
+ mapping.replace(SqlTypeName.ARRAY, arrayMapping);
+ rules = SqlTypeCoercionRule.instance(mapping);
+
+ SqlTypeCoercionRule.THREAD_PROVIDERS.set(rules);
+ } else if (config.typeCoercionRules() != null) {
SqlTypeCoercionRule.THREAD_PROVIDERS.set(config.typeCoercionRules());
}
}
@@ -348,6 +379,11 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
return unknownType;
}
+ @Override
+ public TimeFrameSet getTimeFrameSet() {
+ return timeFrameSet;
+ }
+
@Override
public SqlNodeList expandStar(
SqlNodeList selectList, SqlSelect select, boolean includeSystemVars) {
@@ -387,7 +423,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
SelectScope cursorScope = new SelectScope(parentScope, null, select);
clauseScopes.put(IdPair.of(select, Clause.CURSOR), cursorScope);
final SelectNamespace selectNs = createSelectNamespace(select, select);
- String alias = deriveAlias(select, nextGeneratedId++);
+ final String alias = SqlValidatorUtil.alias(select, nextGeneratedId++);
registerNamespace(cursorScope, alias, selectNs, false);
}
@@ -441,12 +477,12 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
// parentheses-free functions such as LOCALTIME into explicit function
// calls.
SqlNode expanded = expandSelectExpr(selectItem, scope, select);
- final String alias = deriveAliasNonNull(selectItem, aliases.size());
+ final String alias = SqlValidatorUtil.alias(selectItem, aliases.size());
// If expansion has altered the natural alias, supply an explicit 'AS'.
final SqlValidatorScope selectScope = getSelectScope(select);
if (expanded != selectItem) {
- String newAlias = deriveAliasNonNull(expanded, aliases.size());
+ String newAlias = SqlValidatorUtil.alias(expanded, aliases.size());
if (!Objects.equals(newAlias, alias)) {
expanded =
SqlStdOperatorTable.AS.createCall(
@@ -460,9 +496,8 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
selectItems.add(expanded);
aliases.add(alias);
- if (expanded != null) {
- inferUnknownTypes(targetType, scope, expanded);
- }
+ inferUnknownTypes(targetType, selectScope, expanded);
+
RelDataType type = deriveType(selectScope, expanded);
// Re-derive SELECT ITEM's data type that may be nullable in AggregatingSelectScope when it
// appears in advanced grouping elements such as CUBE, ROLLUP , GROUPING SETS.
@@ -486,7 +521,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
if (identifier.getSimple().equals(name)) {
final List<SqlNode> qualifiedNode = new ArrayList<>();
for (ScopeChild child : requireNonNull(scope, "scope").children) {
- if (child.namespace.getRowType().getFieldNames().indexOf(name) >= 0) {
+ if (child.namespace.getRowType().getFieldNames().contains(name)) {
final SqlIdentifier exp =
new SqlIdentifier(
ImmutableList.of(child.name, name),
@@ -1759,7 +1794,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
final List<String> aliasList = new ArrayList<>();
final List<RelDataType> typeList = new ArrayList<>();
for (Ord<SqlNode> column : Ord.zip(rowConstructor.getOperandList())) {
- final String alias = deriveAliasNonNull(column.e, column.i);
+ final String alias = SqlValidatorUtil.alias(column.e, column.i);
aliasList.add(alias);
final RelDataType type = deriveType(scope, column.e);
typeList.add(type);
@@ -2101,7 +2136,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
SqlNode exp,
SelectScope scope,
final boolean includeSystemVars) {
- String alias = SqlValidatorUtil.getAlias(exp, -1);
+ final @Nullable String alias = SqlValidatorUtil.alias(exp);
String uniqueAlias =
SqlValidatorUtil.uniquify(alias, aliases, SqlValidatorUtil.EXPR_SUGGESTER);
if (!Objects.equals(alias, uniqueAlias)) {
@@ -2113,13 +2148,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
@Override
public @Nullable String deriveAlias(SqlNode node, int ordinal) {
- return SqlValidatorUtil.getAlias(node, ordinal);
- }
-
- private String deriveAliasNonNull(SqlNode node, int ordinal) {
- return requireNonNull(
- deriveAlias(node, ordinal),
- () -> "non-null alias expected for node = " + node + ", ordinal = " + ordinal);
+ return ordinal < 0 ? SqlValidatorUtil.alias(node) : SqlValidatorUtil.alias(node, ordinal);
}
protected boolean shouldAllowIntermediateOrderBy() {
@@ -2290,9 +2319,9 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
switch (kind) {
case IDENTIFIER:
case OVER:
- alias = deriveAlias(node, -1);
+ alias = SqlValidatorUtil.alias(node);
if (alias == null) {
- alias = deriveAliasNonNull(node, nextGeneratedId++);
+ alias = SqlValidatorUtil.alias(node, nextGeneratedId++);
}
if (config.identifierExpansion()) {
newNode = SqlValidatorUtil.addAlias(node, alias);
@@ -2313,7 +2342,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
// give this anonymous construct a name since later
// query processing stages rely on it
- alias = deriveAliasNonNull(node, nextGeneratedId++);
+ alias = SqlValidatorUtil.alias(node, nextGeneratedId++);
if (config.identifierExpansion()) {
// Since we're expanding identifiers, we should make the
// aliases explicit too, otherwise the expanded query
@@ -2569,7 +2598,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
case WITH:
case OTHER_FUNCTION:
if (alias == null) {
- alias = deriveAliasNonNull(node, nextGeneratedId++);
+ alias = SqlValidatorUtil.alias(node, nextGeneratedId++);
}
registerQuery(
parentScope,
@@ -2964,7 +2993,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
CollectScope cs = new CollectScope(parentScope, usingScope, call);
final CollectNamespace tableConstructorNs =
new CollectNamespace(call, cs, enclosingNode);
- final String alias2 = deriveAliasNonNull(node, nextGeneratedId++);
+ final String alias2 = SqlValidatorUtil.alias(node, nextGeneratedId++);
registerNamespace(usingScope, alias2, tableConstructorNs, forceNullable);
operands = call.getOperandList();
for (int i = 0; i < operands.size(); i++) {
@@ -3291,6 +3320,19 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
}
}
+ @Override
+ public TimeFrame validateTimeFrame(SqlIntervalQualifier qualifier) {
+ if (qualifier.timeFrameName == null) {
+ final TimeFrame timeFrame = timeFrameSet.get(qualifier.getUnit());
+ return requireNonNull(timeFrame, () -> "time frame for " + qualifier.getUnit());
+ }
+ final @Nullable TimeFrame timeFrame = timeFrameSet.getOpt(qualifier.timeFrameName);
+ if (timeFrame != null) {
+ return timeFrame;
+ }
+ throw newValidationError(qualifier, RESOURCE.invalidTimeFrame(qualifier.timeFrameName));
+ }
+
/**
* Validates the FROM clause of a query, or (recursively) a child node of the FROM clause: AS,
* OVER, JOIN, VALUES, or sub-query.
@@ -3447,9 +3489,9 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
List<SqlIdentifier> list = (List) getCondition(join);
// Parser ensures that using clause is not empty.
- Preconditions.checkArgument(list.size() > 0, "Empty USING clause");
+ Preconditions.checkArgument(!list.isEmpty(), "Empty USING clause");
for (SqlIdentifier id : list) {
- validateCommonJoinColumn(id, left, right, scope);
+ validateCommonJoinColumn(id, left, right, scope, natural);
}
break;
default:
@@ -3467,7 +3509,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
for (String name : deriveNaturalJoinColumnList(join)) {
final SqlIdentifier id =
new SqlIdentifier(name, join.isNaturalNode().getParserPosition());
- validateCommonJoinColumn(id, left, right, scope);
+ validateCommonJoinColumn(id, left, right, scope, natural);
}
}
@@ -3530,13 +3572,20 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
/** Validates a column in a USING clause, or an inferred join key in a NATURAL join. */
private void validateCommonJoinColumn(
- SqlIdentifier id, SqlNode left, SqlNode right, SqlValidatorScope scope) {
+ SqlIdentifier id,
+ SqlNode left,
+ SqlNode right,
+ SqlValidatorScope scope,
+ boolean natural) {
if (id.names.size() != 1) {
throw newValidationError(id, RESOURCE.columnNotFound(id.toString()));
}
- final RelDataType leftColType = validateCommonInputJoinColumn(id, left, scope);
- final RelDataType rightColType = validateCommonInputJoinColumn(id, right, scope);
+ final RelDataType leftColType =
+ natural
+ ? checkAndDeriveDataType(id, left)
+ : validateCommonInputJoinColumn(id, left, scope, natural);
+ final RelDataType rightColType = validateCommonInputJoinColumn(id, right, scope, natural);
if (!SqlTypeUtil.isComparable(leftColType, rightColType)) {
throw newValidationError(
id,
@@ -3545,12 +3594,25 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
}
}
+ private RelDataType checkAndDeriveDataType(SqlIdentifier id, SqlNode node) {
+ Preconditions.checkArgument(id.names.size() == 1);
+ String name = id.names.get(0);
+ SqlNameMatcher nameMatcher = getCatalogReader().nameMatcher();
+ RelDataType rowType = getNamespaceOrThrow(node).getRowType();
+ RelDataType colType =
+ requireNonNull(
+ nameMatcher.field(rowType, name),
+ () -> "unable to find left field " + name + " in " + rowType)
+ .getType();
+ return colType;
+ }
+
/**
* Validates a column in a USING clause, or an inferred join key in a NATURAL join, in the left
* or right input to the join.
*/
private RelDataType validateCommonInputJoinColumn(
- SqlIdentifier id, SqlNode leftOrRight, SqlValidatorScope scope) {
+ SqlIdentifier id, SqlNode leftOrRight, SqlValidatorScope scope, boolean natural) {
Preconditions.checkArgument(id.names.size() == 1);
final String name = id.names.get(0);
final SqlValidatorNamespace namespace = getNamespaceOrThrow(leftOrRight);
@@ -3560,8 +3622,17 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
if (field == null) {
throw newValidationError(id, RESOURCE.columnNotFound(name));
}
- if (nameMatcher.frequency(rowType.getFieldNames(), name) > 1) {
- throw newValidationError(id, RESOURCE.columnInUsingNotUnique(name));
+ Collection<RelDataType> rowTypes;
+ if (!natural && rowType instanceof RelCrossType) {
+ final RelCrossType crossType = (RelCrossType) rowType;
+ rowTypes = new ArrayList<>(crossType.getTypes());
+ } else {
+ rowTypes = Collections.singleton(rowType);
+ }
+ for (RelDataType rowType0 : rowTypes) {
+ if (nameMatcher.frequency(rowType0.getFieldNames(), name) > 1) {
+ throw newValidationError(id, RESOURCE.columnInUsingNotUnique(name));
+ }
}
checkRollUpInUsing(id, leftOrRight, scope);
return field.getType();
@@ -3758,7 +3829,8 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
String context =
contextClause != null ? contextClause : parent.getKind().toString();
throw newValidationError(
- id, RESOURCE.rolledUpNotAllowed(deriveAliasNonNull(id, 0), context));
+ id,
+ RESOURCE.rolledUpNotAllowed(SqlValidatorUtil.alias(id, 0), context));
}
}
}
@@ -4491,6 +4563,15 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
}
}
+ // Unless 'naked measures' are enabled, a non-aggregating query cannot
+ // reference measure columns. (An aggregating query can use them as
+ // argument to the AGGREGATE function.)
+ if (!config.nakedMeasures()
+ && !(scope instanceof AggregatingScope)
+ && scope.isMeasureRef(expr)) {
+ throw newValidationError(expr, RESOURCE.measureMustBeInAggregateQuery());
+ }
+
// Call on the expression to validate itself.
expr.validateExpr(this, scope);
@@ -4524,7 +4605,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
expandedSelectItems.add(selectItem);
// Get or generate alias and add to list.
- final String alias = deriveAliasNonNull(selectItem, aliasList.size());
+ final String alias = SqlValidatorUtil.alias(selectItem, aliasList.size());
aliasList.add(alias);
final SelectScope scope = (SelectScope) getWhereScope(parentSelect);
@@ -5539,7 +5620,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
for (SqlNode measure : measures) {
assert measure instanceof SqlCall;
- final String alias = deriveAliasNonNull(measure, aliases.size());
+ final String alias = SqlValidatorUtil.alias(measure, aliases.size());
aliases.add(alias);
SqlNode expand = expand(measure, scope);
@@ -6025,6 +6106,27 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
assert feature.getProperties().get("FeatureDefinition") != null;
}
+ @Override
+ public SqlLiteral resolveLiteral(SqlLiteral literal) {
+ switch (literal.getTypeName()) {
+ case UNKNOWN:
+ final SqlUnknownLiteral unknownLiteral = (SqlUnknownLiteral) literal;
+ final SqlIdentifier identifier =
+ new SqlIdentifier(unknownLiteral.tag, SqlParserPos.ZERO);
+ final @Nullable RelDataType type = catalogReader.getNamedType(identifier);
+ final SqlTypeName typeName;
+ if (type != null) {
+ typeName = type.getSqlTypeName();
+ } else {
+ typeName = SqlTypeName.lookup(unknownLiteral.tag);
+ }
+ return unknownLiteral.resolve(typeName);
+
+ default:
+ return literal;
+ }
+ }
+
public SqlNode expandSelectExpr(SqlNode expr, SelectScope scope, SqlSelect select) {
final Expander expander = new SelectExpander(this, scope, select);
final SqlNode newExpr = expander.go(expr);
@@ -6088,13 +6190,32 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
final SqlQualified qualified = scope.fullyQualify((SqlIdentifier) selectItem);
SqlValidatorNamespace namespace =
requireNonNull(qualified.namespace, () -> "namespace for " + qualified);
+ if (namespace.isWrapperFor(AliasNamespace.class)) {
+ AliasNamespace aliasNs = namespace.unwrap(AliasNamespace.class);
+ SqlNode aliased =
+ requireNonNull(
+ aliasNs.getNode(), () -> "sqlNode for aliasNs " + aliasNs);
+ namespace = getNamespaceOrThrow(stripAs(aliased));
+ }
+
final SqlValidatorTable table = namespace.getTable();
if (table == null) {
return null;
}
final List<String> origin = new ArrayList<>(table.getQualifiedName());
for (String name : qualified.suffix()) {
+ if (namespace.isWrapperFor(UnnestNamespace.class)) {
+ // If identifier is drawn from a repeated subrecord via unnest, add name of
+ // array field
+ UnnestNamespace unnestNamespace = namespace.unwrap(UnnestNamespace.class);
+ final SqlQualified columnUnnestedFrom =
+ unnestNamespace.getColumnUnnestedFrom(name);
+ if (columnUnnestedFrom != null) {
+ origin.addAll(columnUnnestedFrom.suffix());
+ }
+ }
namespace = namespace.lookupChild(name);
+
if (namespace == null) {
return null;
}
@@ -6329,7 +6450,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
@Override
public RelDataType visit(SqlLiteral literal) {
- return literal.createSqlType(typeFactory);
+ return resolveLiteral(literal).createSqlType(typeFactory);
}
@Override
@@ -6478,6 +6599,11 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
return expandedExpr;
}
+ @Override
+ public @Nullable SqlNode visit(SqlLiteral literal) {
+ return validator.resolveLiteral(literal);
+ }
+
@Override
protected SqlNode visitScoped(SqlCall call) {
switch (call.getKind()) {
@@ -6669,7 +6795,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
final SqlNameMatcher nameMatcher = validator.catalogReader.nameMatcher();
int n = 0;
for (SqlNode s : SqlNonNullableAccessors.getSelectList(select)) {
- final String alias = SqlValidatorUtil.getAlias(s, -1);
+ final @Nullable String alias = SqlValidatorUtil.alias(s);
if (alias != null && nameMatcher.matches(alias, name)) {
expr = s;
n++;
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
index 8aed985a7b0..bc13de49927 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
@@ -1822,10 +1822,11 @@ public class RelDecorrelator implements ReflectiveVisitor {
@Override
public RexNode visitLiteral(RexLiteral literal) {
// Use nullIndicator to decide whether to project null.
- // Do nothing if the literal is null.
+ // Do nothing if the literal is null or symbol.
if (!RexUtil.isNull(literal)
&& projectPulledAboveLeftCorrelator
- && (nullIndicator != null)) {
+ && (nullIndicator != null)
+ && !RexUtil.isSymbolLiteral(literal)) {
return createCaseExpression(nullIndicator, null, literal);
}
return literal;
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index cc9331c49be..e1111ef812f 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -26,12 +26,12 @@ import org.apache.flink.table.planner.plan.FlinkCalciteCatalogSnapshotReader;
import org.apache.flink.table.planner.plan.utils.FlinkRelOptUtil;
import org.apache.flink.table.planner.utils.ShortcutUtils;
-import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import org.apache.calcite.avatica.util.Spaces;
+import org.apache.calcite.config.NullCollation;
import org.apache.calcite.jdbc.CalciteSchema;
import org.apache.calcite.linq4j.Ord;
import org.apache.calcite.linq4j.tree.TableExpressionFactory;
@@ -83,6 +83,7 @@ import org.apache.calcite.rel.logical.LogicalUnion;
import org.apache.calcite.rel.logical.LogicalValues;
import org.apache.calcite.rel.metadata.RelColumnMapping;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.rel2sql.SqlImplementor;
import org.apache.calcite.rel.stream.Delta;
import org.apache.calcite.rel.stream.LogicalDelta;
import org.apache.calcite.rel.type.RelDataType;
@@ -221,6 +222,7 @@ import java.util.function.Supplier;
import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
+import static com.google.common.base.Preconditions.checkArgument;
import static java.util.Objects.requireNonNull;
import static org.apache.calcite.linq4j.Nullness.castNonNull;
import static org.apache.calcite.sql.SqlUtil.stripAs;
@@ -235,18 +237,18 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
* <p>FLINK modifications are at lines
*
* <ol>
- * <li>Added in FLINK-29081, FLINK-28682, FLINK-33395: Lines 658 ~ 675
- * <li>Added in Flink-24024: Lines 1439 ~ 1449
- * <li>Added in Flink-24024: Lines 1463 ~ 1506
- * <li>Added in FLINK-28682: Lines 2328 ~ 2345
- * <li>Added in FLINK-28682: Lines 2382 ~ 2410
- * <li>Added in FLINK-32474: Lines 2462 ~ 2464
- * <li>Added in FLINK-32474: Lines 2468 ~ 2470
- * <li>Added in FLINK-32474: Lines 2481 ~ 2483
- * <li>Added in FLINK-32474: Lines 2886 ~ 2898
- * <li>Added in FLINK-32474: Lines 2998 ~ 3032
- * <li>Added in FLINK-20873: Lines 5529 ~ 5538
- * <li>Added in FLINK-34057, FLINK-34058: Lines 6099 ~ 6102
+ * <li>Added in FLINK-29081, FLINK-28682, FLINK-33395: Lines 661 ~ 678
+ * <li>Added in Flink-24024: Lines 1453 ~ 1463
+ * <li>Added in Flink-24024: Lines 1477 ~ 1520
+ * <li>Added in FLINK-28682: Lines 2346 ~ 2363
+ * <li>Added in FLINK-28682: Lines 2400 ~ 2428
+ * <li>Added in FLINK-32474: Lines 2480 ~ 2482
+ * <li>Added in FLINK-32474: Lines 2486 ~ 2488
+ * <li>Added in FLINK-32474: Lines 2499 ~ 2501
+ * <li>Added in FLINK-32474: Lines 2906 ~ 2918
+ * <li>Added in FLINK-32474: Lines 3019 ~ 3053
+ * <li>Added in FLINK-34312: Lines 5693 ~ 5696
+ * <li>Added in FLINK-34057, FLINK-34058, FLINK-34312: Lines 6144 ~ 6162
* </ol>
*/
@SuppressWarnings("UnstableApiUsage")
@@ -862,7 +864,8 @@ public class SqlToRelConverter {
rel,
ImmutableList.of(),
Pair.left(newProjects),
- Pair.right(newProjects));
+ Pair.right(newProjects),
+ project.getVariablesSet());
bb.root = rel;
distinctify(bb, false);
rel = bb.root();
@@ -884,7 +887,8 @@ public class SqlToRelConverter {
rel,
ImmutableList.of(),
Pair.left(undoProjects),
- Pair.right(undoProjects));
+ Pair.right(undoProjects),
+ ImmutableSet.of());
bb.setRoot(rel, false);
return;
@@ -960,7 +964,8 @@ public class SqlToRelConverter {
bb.root(),
ImmutableList.of(),
exprs,
- rowType.getFieldNames().subList(0, fieldCount)),
+ rowType.getFieldNames().subList(0, fieldCount),
+ ImmutableSet.of()),
false);
}
}
@@ -1159,7 +1164,15 @@ public class SqlToRelConverter {
private void replaceSubQueries(
final Blackboard bb, final SqlNode expr, RelOptUtil.Logic logic) {
- findSubQueries(bb, expr, logic, false);
+ replaceSubQueries(bb, expr, logic, null);
+ }
+
+ private void replaceSubQueries(
+ final Blackboard bb,
+ final SqlNode expr,
+ RelOptUtil.Logic logic,
+ final SqlImplementor.Clause clause) {
+ findSubQueries(bb, expr, logic, false, clause);
for (SubQuery node : bb.subQueryList) {
substituteSubQuery(bb, node);
}
@@ -2062,12 +2075,14 @@ public class SqlToRelConverter {
* FALSE)
* @param registerOnlyScalarSubQueries if set to true and the parse tree corresponds to a
* variation of a select node, only register it if it's a scalar sub-query
+ * @param clause A clause inside which sub-query is searched
*/
private void findSubQueries(
Blackboard bb,
SqlNode node,
RelOptUtil.Logic logic,
- boolean registerOnlyScalarSubQueries) {
+ boolean registerOnlyScalarSubQueries,
+ SqlImplementor.Clause clause) {
final SqlKind kind = node.getKind();
switch (kind) {
case EXISTS:
@@ -2081,7 +2096,7 @@ public class SqlToRelConverter {
case SET_SEMANTICS_TABLE:
case SCALAR_QUERY:
if (!registerOnlyScalarSubQueries || (kind == SqlKind.SCALAR_QUERY)) {
- bb.registerSubQuery(node, RelOptUtil.Logic.TRUE_FALSE);
+ bb.registerSubQuery(node, RelOptUtil.Logic.TRUE_FALSE, clause);
}
return;
case IN:
@@ -2118,7 +2133,8 @@ public class SqlToRelConverter {
|| kind == SqlKind.NOT_IN
|| kind == SqlKind.SOME
|| kind == SqlKind.ALL
- || registerOnlyScalarSubQueries);
+ || registerOnlyScalarSubQueries,
+ clause);
}
}
} else if (node instanceof SqlNodeList) {
@@ -2131,7 +2147,8 @@ public class SqlToRelConverter {
|| kind == SqlKind.NOT_IN
|| kind == SqlKind.SOME
|| kind == SqlKind.ALL
- || registerOnlyScalarSubQueries);
+ || registerOnlyScalarSubQueries,
+ clause);
}
}
@@ -2161,7 +2178,7 @@ public class SqlToRelConverter {
default:
break;
}
- bb.registerSubQuery(node, logic);
+ bb.registerSubQuery(node, logic, clause);
break;
default:
break;
@@ -2294,7 +2311,7 @@ public class SqlToRelConverter {
}
try {
- Preconditions.checkArgument(bb.window == null, "already in window agg mode");
+ checkArgument(bb.window == null, "already in window agg mode");
bb.window = window;
RexNode rexAgg = exprConverter.convertCall(bb, aggCall);
rexAgg = rexBuilder.ensureType(validator().getValidatedNodeType(call), rexAgg, false);
@@ -2532,11 +2549,13 @@ public class SqlToRelConverter {
replaceSubQueries(bb, node, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
}
final List<RexNode> exprs = new ArrayList<>();
- for (Ord<SqlNode> node : Ord.zip(nodes)) {
- exprs.add(
- relBuilder.alias(
- bb.convertExpression(node.e), validator().deriveAlias(node.e, node.i)));
- }
+ Ord.forEach(
+ nodes,
+ (node, i) -> {
+ final RexNode e = bb.convertExpression(node);
+ final String alias = SqlValidatorUtil.alias(node, i);
+ exprs.add(relBuilder.alias(e, alias));
+ });
RelNode child = (null != bb.root) ? bb.root : LogicalValues.createOneRow(cluster);
RelNode uncollect;
if (validator().config().conformance().allowAliasUnnestItems()) {
@@ -2901,7 +2920,8 @@ public class SqlToRelConverter {
if (extendedColumns != null && extendedColumns.size() > 0) {
final SqlValidatorTable validatorTable = table.unwrapOrThrow(SqlValidatorTable.class);
final List<RelDataTypeField> extendedFields =
- SqlValidatorUtil.getExtendedColumns(validator, validatorTable, extendedColumns);
+ SqlValidatorUtil.getExtendedColumns(
+ validator(), validatorTable, extendedColumns);
table = table.extend(extendedFields);
}
// Review Danny 2020-01-13: hacky to construct a new table scan
@@ -3525,13 +3545,13 @@ public class SqlToRelConverter {
// also replace sub-queries inside ordering spec in the aggregates
replaceSubQueries(bb, aggregateFinder.orderList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
-
// If group-by clause is missing, pretend that it has zero elements.
if (groupList == null) {
groupList = SqlNodeList.EMPTY;
}
- replaceSubQueries(bb, groupList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+ replaceSubQueries(
+ bb, groupList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN, SqlImplementor.Clause.GROUP_BY);
// register the group exprs
@@ -3552,7 +3572,7 @@ public class SqlToRelConverter {
final List<Pair<RexNode, String>> projects = new ArrayList<>();
try {
- Preconditions.checkArgument(bb.agg == null, "already in agg mode");
+ checkArgument(bb.agg == null, "already in agg mode");
bb.agg = aggConverter;
// convert the select and having expressions, so that the
@@ -3623,7 +3643,11 @@ public class SqlToRelConverter {
// This needs to be done separately from the sub-query inside
// any aggregate in the select list, and after the aggregate rel
// is allocated.
- replaceSubQueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
+ replaceSubQueries(
+ bb,
+ selectList,
+ RelOptUtil.Logic.TRUE_FALSE_UNKNOWN,
+ SqlImplementor.Clause.SELECT);
// Now sub-queries in the entire select list have been converted.
// Convert the select expressions to get the final list to be
@@ -3647,15 +3671,13 @@ public class SqlToRelConverter {
Pair.of(
bb.convertExpression(expr),
k < sysFieldCount
- ? castNonNull(validator().deriveAlias(expr, k++))
+ ? SqlValidatorUtil.alias(expr, k++)
: names.get(k++ - sysFieldCount)));
}
for (SqlNode expr : orderExprList) {
projects.add(
- Pair.of(
- bb.convertExpression(expr),
- castNonNull(validator().deriveAlias(expr, k++))));
+ Pair.of(bb.convertExpression(expr), SqlValidatorUtil.alias(expr, k++)));
}
} finally {
bb.agg = null;
@@ -4523,7 +4545,7 @@ public class SqlToRelConverter {
* @return Relational expression which returns a single row.
*/
private RelNode convertRowConstructor(Blackboard bb, SqlCall rowConstructor) {
- Preconditions.checkArgument(isRowConstructor(rowConstructor));
+ checkArgument(isRowConstructor(rowConstructor));
final List<SqlNode> operands = rowConstructor.getOperandList();
return convertMultisets(operands, bb);
}
@@ -4604,7 +4626,7 @@ public class SqlToRelConverter {
Collect.create(
requireNonNull(input, "input"),
call.getKind(),
- castNonNull(validator().deriveAlias(call, i))));
+ SqlValidatorUtil.alias(call, i)));
joinList.add(relBuilder.build());
}
@@ -4699,7 +4721,16 @@ public class SqlToRelConverter {
final RelNode r;
final CorrelationUse p = getCorrelationUse(bb, project);
if (p != null) {
- r = p.r;
+ assert p.r instanceof Project;
+ // correlation variables have been normalized in p.r, we should use expressions
+ // in p.r instead of the original exprs
+ Project project1 = (Project) p.r;
+ r =
+ relBuilder
+ .push(bb.root())
+ .projectNamed(
+ project1.getProjects(), fieldNames, true, ImmutableSet.of(p.id))
+ .build();
} else {
r = project;
}
@@ -4732,10 +4763,12 @@ public class SqlToRelConverter {
Collection<String> aliasList,
List<SqlMonotonicity> columnMonotonicityList) {}
- private String deriveAlias(final SqlNode node, Collection<String> aliases, final int ordinal) {
- String alias = validator().deriveAlias(node, ordinal);
- if (alias == null || aliases.contains(alias)) {
- final String aliasBase = Util.first(alias, SqlUtil.GENERATED_EXPR_ALIAS_PREFIX);
+ private static String deriveAlias(
+ final SqlNode node, Collection<String> aliases, final int ordinal) {
+ checkArgument(ordinal >= 0);
+ String alias = SqlValidatorUtil.alias(node, ordinal);
+ if (aliases.contains(alias)) {
+ final String aliasBase = alias;
for (int j = 0; ; j++) {
alias = aliasBase + j;
if (!aliases.contains(alias)) {
@@ -4785,12 +4818,13 @@ public class SqlToRelConverter {
Blackboard tmpBb = createBlackboard(bb.scope, null, false);
replaceSubQueries(tmpBb, rowConstructor, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
final List<Pair<RexNode, String>> exps = new ArrayList<>();
- for (Ord<SqlNode> operand : Ord.zip(rowConstructor.getOperandList())) {
- exps.add(
- Pair.of(
- tmpBb.convertExpression(operand.e),
- castNonNull(validator().deriveAlias(operand.e, operand.i))));
- }
+ Ord.forEach(
+ rowConstructor.getOperandList(),
+ (operand, i) ->
+ exps.add(
+ Pair.of(
+ tmpBb.convertExpression(operand),
+ SqlValidatorUtil.alias(operand, i))));
RelNode in = (null == tmpBb.root) ? LogicalValues.createOneRow(cluster) : tmpBb.root;
relBuilder.push(in).project(Pair.left(exps), Pair.right(exps));
}
@@ -5217,27 +5251,31 @@ public class SqlToRelConverter {
}
}
- void registerSubQuery(SqlNode node, RelOptUtil.Logic logic) {
- for (SubQuery subQuery : subQueryList) {
- // Compare the reference to make sure the matched node has
- // exact scope where it belongs.
- if (node == subQuery.node) {
- return;
- }
+ void registerSubQuery(SqlNode node, RelOptUtil.Logic logic, SqlImplementor.Clause clause) {
+ if (getSubQuery(node, clause) == null) {
+ subQueryList.add(new SubQuery(node, logic, clause));
}
- subQueryList.add(new SubQuery(node, logic));
}
@Nullable
- SubQuery getSubQuery(SqlNode expr) {
+ SubQuery getSubQuery(SqlNode expr, SqlImplementor.Clause exprClause) {
for (SubQuery subQuery : subQueryList) {
// Compare the reference to make sure the matched node has
// exact scope where it belongs.
if (expr == subQuery.node) {
return subQuery;
}
- }
+ // Reference comparing does not work in case when select list has column which
+ // refers
+ // to the column inside `GROUP BY` clause.
+ // For example: SELECT deptno IN (1,2) FROM emp.deptno GROUP BY deptno IN (1,2);
+ if (exprClause == SqlImplementor.Clause.SELECT
+ && subQuery.clause == SqlImplementor.Clause.GROUP_BY
+ && expr.equalsDeep(subQuery.node, Litmus.IGNORE)) {
+ return subQuery;
+ }
+ }
return null;
}
@@ -5392,7 +5430,7 @@ public class SqlToRelConverter {
case CURSOR:
case IN:
case NOT_IN:
- subQuery = requireNonNull(getSubQuery(expr));
+ subQuery = requireNonNull(getSubQuery(expr, null));
rex = requireNonNull(subQuery.expr);
return StandardConvertletTable.castToValidatedType(
expr, rex, validator(), rexBuilder);
@@ -5403,7 +5441,7 @@ public class SqlToRelConverter {
case ARRAY_QUERY_CONSTRUCTOR:
case MAP_QUERY_CONSTRUCTOR:
case MULTISET_QUERY_CONSTRUCTOR:
- subQuery = getSubQuery(expr);
+ subQuery = getSubQuery(expr, null);
assert subQuery != null;
rex = subQuery.expr;
assert rex != null : "rex != null";
@@ -5444,6 +5482,7 @@ public class SqlToRelConverter {
* Converts an item in an ORDER BY clause inside a window (OVER) clause, extracting DESC,
* NULLS LAST and NULLS FIRST flags first.
*/
+ @Deprecated // to be removed before 2.0
public RexFieldCollation convertSortExpression(
SqlNode expr,
RelFieldCollation.Direction direction,
@@ -5485,6 +5524,8 @@ public class SqlToRelConverter {
}
}
+ // Only used by deprecated method "convertSortExpression", and will be
+ // removed with that method.
private RexFieldCollation sortToRexFieldCollation(
SqlNode expr,
RelFieldCollation.Direction direction,
@@ -5524,19 +5565,21 @@ public class SqlToRelConverter {
RelFieldCollation.Direction direction,
RelFieldCollation.NullDirection nullDirection) {
RexNode node = convertExpression(expr);
- if (direction == RelFieldCollation.Direction.DESCENDING) {
+ final boolean desc = direction == RelFieldCollation.Direction.DESCENDING;
+ if (desc) {
node = relBuilder.desc(node);
}
- // ----- FLINK MODIFICATION BEGIN -----
- // if null direction is unspecified then check default
- // to keep same behavior as before Calcite 1.27.0
if (nullDirection == RelFieldCollation.NullDirection.UNSPECIFIED) {
- nullDirection =
- validator().config().defaultNullCollation().last(desc(direction))
- ? RelFieldCollation.NullDirection.LAST
- : RelFieldCollation.NullDirection.FIRST;
+ final NullCollation nullCollation = validator().config().defaultNullCollation();
+ final boolean nullsLast = nullCollation.last(desc);
+ final boolean nullsFirst = !nullsLast;
+ if (!NullCollation.HIGH.isDefaultOrder(nullsFirst, desc)) {
+ nullDirection =
+ nullsLast
+ ? RelFieldCollation.NullDirection.LAST
+ : RelFieldCollation.NullDirection.FIRST;
+ }
}
- // ----- FLINK MODIFICATION END -----
if (nullDirection == RelFieldCollation.NullDirection.FIRST) {
node = relBuilder.nullsFirst(node);
}
@@ -5599,7 +5642,7 @@ public class SqlToRelConverter {
@Override
public RexRangeRef getSubQueryExpr(SqlCall call) {
- final SubQuery subQuery = getSubQuery(call);
+ final SubQuery subQuery = getSubQuery(call, null);
assert subQuery != null;
return (RexRangeRef) requireNonNull(subQuery.expr, () -> "subQuery.expr for " + call);
}
@@ -5808,8 +5851,7 @@ public class SqlToRelConverter {
name = call.operand(1).toString();
}
if (name == null) {
- name = validator().deriveAlias(selectItem, i);
- assert name != null : "alias must not be null for " + selectItem + ", i=" + i;
+ name = SqlValidatorUtil.alias(selectItem, i);
}
nameMap.put(selectItem.toString(), name);
}
@@ -6515,10 +6557,12 @@ public class SqlToRelConverter {
final SqlNode node;
final RelOptUtil.Logic logic;
@Nullable RexNode expr;
+ final SqlImplementor.Clause clause;
- private SubQuery(SqlNode node, RelOptUtil.Logic logic) {
+ private SubQuery(SqlNode node, RelOptUtil.Logic logic, SqlImplementor.Clause clause) {
this.node = node;
this.logic = logic;
+ this.clause = clause;
}
}
@@ -6815,7 +6859,8 @@ public class SqlToRelConverter {
newInput,
project.getHints(),
newProjections.build(),
- project.getRowType().getFieldNames());
+ project.getRowType().getFieldNames(),
+ project.getVariablesSet());
}
private Set<Integer> requiredJsonOutputFromParent(RelNode relNode) {
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java
index 07b2810c541..71580fb2ea9 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java
@@ -25,6 +25,7 @@ import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeFamily;
import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rel.type.TimeFrame;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexCallBinding;
@@ -59,6 +60,7 @@ import org.apache.calcite.sql.fun.SqlBetweenOperator;
import org.apache.calcite.sql.fun.SqlCase;
import org.apache.calcite.sql.fun.SqlDatetimeSubtractionOperator;
import org.apache.calcite.sql.fun.SqlExtractFunction;
+import org.apache.calcite.sql.fun.SqlInternalOperators;
import org.apache.calcite.sql.fun.SqlJsonQueryFunction;
import org.apache.calcite.sql.fun.SqlJsonValueFunction;
import org.apache.calcite.sql.fun.SqlLibrary;
@@ -95,11 +97,16 @@ import java.util.stream.Collectors;
import static java.util.Objects.requireNonNull;
import static org.apache.calcite.sql.type.NonNullableAccessors.getComponentTypeOrThrow;
+import static org.apache.calcite.util.Util.first;
/**
* Standard implementation of {@link SqlRexConvertletTable}.
*
- * <p>Lines 691-736 implement supporting RETURNING clause in JSON_QUERY (CALCITE-6365).
+ * <p>FLINK modifications are at lines
+ *
+ * <ol>
+ * <li>Added in Flink-35216: Lines 712 ~ 757
+ * </ol>
*/
public class StandardConvertletTable extends ReflectiveConvertletTable {
@@ -113,11 +120,13 @@ public class StandardConvertletTable extends ReflectiveConvertletTable {
// Register aliases (operators which have a different name but
// identical behavior to other operators).
+ addAlias(SqlLibraryOperators.LENGTH, SqlStdOperatorTable.CHAR_LENGTH);
addAlias(SqlStdOperatorTable.CHARACTER_LENGTH, SqlStdOperatorTable.CHAR_LENGTH);
addAlias(SqlStdOperatorTable.IS_UNKNOWN, SqlStdOperatorTable.IS_NULL);
addAlias(SqlStdOperatorTable.IS_NOT_UNKNOWN, SqlStdOperatorTable.IS_NOT_NULL);
addAlias(SqlLibraryOperators.NULL_SAFE_EQUAL, SqlStdOperatorTable.IS_NOT_DISTINCT_FROM);
addAlias(SqlStdOperatorTable.PERCENT_REMAINDER, SqlStdOperatorTable.MOD);
+ addAlias(SqlLibraryOperators.IFNULL, SqlLibraryOperators.NVL);
// Register convertlets for specific objects.
registerOp(SqlStdOperatorTable.CAST, this::convertCast);
@@ -157,6 +166,14 @@ public class StandardConvertletTable extends ReflectiveConvertletTable {
registerOp(
SqlLibraryOperators.SUBSTR_POSTGRESQL, new SubstrConvertlet(SqlLibrary.POSTGRESQL));
+ registerOp(SqlLibraryOperators.DATE_SUB, new TimestampSubConvertlet());
+ registerOp(SqlLibraryOperators.TIME_ADD, new TimestampAddConvertlet());
+ registerOp(SqlLibraryOperators.TIME_DIFF, new TimestampDiffConvertlet());
+ registerOp(SqlLibraryOperators.TIME_SUB, new TimestampSubConvertlet());
+ registerOp(SqlLibraryOperators.TIMESTAMP_ADD2, new TimestampAddConvertlet());
+ registerOp(SqlLibraryOperators.TIMESTAMP_DIFF3, new TimestampDiffConvertlet());
+ registerOp(SqlLibraryOperators.TIMESTAMP_SUB, new TimestampSubConvertlet());
+
registerOp(SqlLibraryOperators.NVL, StandardConvertletTable::convertNvl);
registerOp(SqlLibraryOperators.DECODE, StandardConvertletTable::convertDecode);
registerOp(SqlLibraryOperators.IF, StandardConvertletTable::convertIf);
@@ -261,7 +278,6 @@ public class StandardConvertletTable extends ReflectiveConvertletTable {
final SqlRexConvertlet floorCeilConvertlet = new FloorCeilConvertlet();
registerOp(SqlStdOperatorTable.FLOOR, floorCeilConvertlet);
registerOp(SqlStdOperatorTable.CEIL, floorCeilConvertlet);
-
registerOp(SqlStdOperatorTable.TIMESTAMP_ADD, new TimestampAddConvertlet());
registerOp(SqlStdOperatorTable.TIMESTAMP_DIFF, new TimestampDiffConvertlet());
@@ -303,7 +319,7 @@ public class StandardConvertletTable extends ReflectiveConvertletTable {
}
}
- /** Converts a call to the NVL function. */
+ /** Converts a call to the {@code NVL} function (and also its synonym, {@code IFNULL}). */
private static RexNode convertNvl(SqlRexContext cx, SqlCall call) {
final RexBuilder rexBuilder = cx.getRexBuilder();
final RexNode operand0 = cx.convertExpression(call.getOperandList().get(0));
@@ -554,9 +570,14 @@ public class StandardConvertletTable extends ReflectiveConvertletTable {
return cx.convertExpression(left);
}
if (null != dataType.getCollectionsTypeName()) {
- final RelDataType argComponentType =
- requireNonNull(
- arg.getType().getComponentType(), () -> "componentType of " + arg);
+ RelDataType argComponentType = arg.getType().getComponentType();
+
+ // arg.getType() may be ANY
+ if (argComponentType == null) {
+ argComponentType = dataType.getComponentTypeSpec().deriveType(cx.getValidator());
+ }
+
+ requireNonNull(argComponentType, () -> "componentType of " + arg);
RelDataType typeFinal = type;
final RelDataType componentType =
@@ -1767,19 +1788,57 @@ public class StandardConvertletTable extends ReflectiveConvertletTable {
}
}
- /** Convertlet that handles the {@code TIMESTAMPADD} function. */
+ /**
+ * Convertlet that handles the 3-argument {@code TIMESTAMPADD} function and the 2-argument
+ * BigQuery-style {@code TIMESTAMP_ADD} function.
+ */
private static class TimestampAddConvertlet implements SqlRexConvertlet {
@Override
public RexNode convertCall(SqlRexContext cx, SqlCall call) {
// TIMESTAMPADD(unit, count, timestamp)
// => timestamp + count * INTERVAL '1' UNIT
final RexBuilder rexBuilder = cx.getRexBuilder();
- final SqlLiteral unitLiteral = call.operand(0);
- final TimeUnit unit = unitLiteral.getValueAs(TimeUnit.class);
+ SqlIntervalQualifier qualifier;
+ final RexNode op1;
+ final RexNode op2;
+ switch (call.operandCount()) {
+ case 2:
+ // BigQuery-style 'TIMESTAMP_ADD(timestamp, interval)'
+ final SqlBasicCall operandCall = call.operand(1);
+ qualifier = operandCall.operand(1);
+ op1 = cx.convertExpression(operandCall.operand(0));
+ op2 = cx.convertExpression(call.operand(0));
+ break;
+ default:
+ // JDBC-style 'TIMESTAMPADD(unit, count, timestamp)'
+ qualifier = call.operand(0);
+ op1 = cx.convertExpression(call.operand(1));
+ op2 = cx.convertExpression(call.operand(2));
+ }
+
+ final TimeFrame timeFrame = cx.getValidator().validateTimeFrame(qualifier);
+ final TimeUnit unit = first(timeFrame.unit(), TimeUnit.EPOCH);
+ final RelDataType type = cx.getValidator().getValidatedNodeType(call);
+ if (unit == TimeUnit.EPOCH && qualifier.timeFrameName != null) {
+ // Custom time frames have a different path. They are kept as names,
+ // and then handled by Java functions such as
+ // SqlFunctions.customTimestampAdd.
+ final RexLiteral timeFrameName = rexBuilder.makeLiteral(qualifier.timeFrameName);
+ // If the TIMESTAMPADD call has type TIMESTAMP and op2 has type DATE
+ // (which can happen for sub-day time frames such as HOUR), cast op2 to
+ // TIMESTAMP.
+ final RexNode op2b = rexBuilder.makeCast(type, op2, false);
+ return rexBuilder.makeCall(
+ type,
+ SqlStdOperatorTable.TIMESTAMP_ADD,
+ ImmutableList.of(timeFrameName, op1, op2b));
+ }
+
+ if (qualifier.getUnit() != unit) {
+ qualifier = new SqlIntervalQualifier(unit, null, qualifier.getParserPosition());
+ }
+
RexNode interval2Add;
- SqlIntervalQualifier qualifier =
- new SqlIntervalQualifier(unit, null, unitLiteral.getParserPosition());
- RexNode op1 = cx.convertExpression(call.operand(1));
switch (unit) {
case MICROSECOND:
case NANOSECOND:
@@ -1802,10 +1861,49 @@ public class StandardConvertletTable extends ReflectiveConvertletTable {
op1);
}
- return rexBuilder.makeCall(
- SqlStdOperatorTable.DATETIME_PLUS,
- cx.convertExpression(call.operand(2)),
- interval2Add);
+ return rexBuilder.makeCall(SqlStdOperatorTable.DATETIME_PLUS, op2, interval2Add);
+ }
+ }
+
+ /** Convertlet that handles the BigQuery {@code TIMESTAMP_SUB} function. */
+ private static class TimestampSubConvertlet implements SqlRexConvertlet {
+ @Override
+ public RexNode convertCall(SqlRexContext cx, SqlCall call) {
+ // TIMESTAMP_SUB(timestamp, interval)
+ // => timestamp - count * INTERVAL '1' UNIT
+ final RexBuilder rexBuilder = cx.getRexBuilder();
+ final SqlBasicCall operandCall = call.operand(1);
+ SqlIntervalQualifier qualifier = operandCall.operand(1);
+ final RexNode op1 = cx.convertExpression(operandCall.operand(0));
+ final RexNode op2 = cx.convertExpression(call.operand(0));
+ final TimeFrame timeFrame = cx.getValidator().validateTimeFrame(qualifier);
+ final TimeUnit unit = first(timeFrame.unit(), TimeUnit.EPOCH);
+ final RexNode interval2Sub;
+ switch (unit) {
+ // Fractional second units are converted to seconds using their associated
+ // multiplier.
+ case MICROSECOND:
+ case NANOSECOND:
+ interval2Sub =
+ divide(
+ rexBuilder,
+ multiply(
+ rexBuilder,
+ rexBuilder.makeIntervalLiteral(
+ BigDecimal.ONE, qualifier),
+ op1),
+ BigDecimal.ONE.divide(
+ unit.multiplier, RoundingMode.UNNECESSARY));
+ break;
+ default:
+ interval2Sub =
+ multiply(
+ rexBuilder,
+ rexBuilder.makeIntervalLiteral(unit.multiplier, qualifier),
+ op1);
+ }
+
+ return rexBuilder.makeCall(SqlInternalOperators.MINUS_DATE2, op2, interval2Sub);
}
}
@@ -1815,9 +1913,34 @@ public class StandardConvertletTable extends ReflectiveConvertletTable {
public RexNode convertCall(SqlRexContext cx, SqlCall call) {
// TIMESTAMPDIFF(unit, t1, t2)
// => (t2 - t1) UNIT
+ // TIMESTAMP_DIFF(t1, t2, unit)
+ // => (t1 - t2) UNIT
+ SqlIntervalQualifier qualifier;
+ final RexNode op1;
+ final RexNode op2;
+ if (call.operand(0).getKind() == SqlKind.INTERVAL_QUALIFIER) {
+ qualifier = call.operand(0);
+ op1 = cx.convertExpression(call.operand(1));
+ op2 = cx.convertExpression(call.operand(2));
+ } else {
+ qualifier = call.operand(2);
+ op1 = cx.convertExpression(call.operand(1));
+ op2 = cx.convertExpression(call.operand(0));
+ }
final RexBuilder rexBuilder = cx.getRexBuilder();
- final SqlLiteral unitLiteral = call.operand(0);
- TimeUnit unit = unitLiteral.getValueAs(TimeUnit.class);
+ final TimeFrame timeFrame = cx.getValidator().validateTimeFrame(qualifier);
+ final TimeUnit unit = first(timeFrame.unit(), TimeUnit.EPOCH);
+
+ if (unit == TimeUnit.EPOCH && qualifier.timeFrameName != null) {
+ // Custom time frames have a different path. They are kept as names, and
+ // then handled by Java functions.
+ final RexLiteral timeFrameName = rexBuilder.makeLiteral(qualifier.timeFrameName);
+ return rexBuilder.makeCall(
+ cx.getValidator().getValidatedNodeType(call),
+ SqlStdOperatorTable.TIMESTAMP_DIFF,
+ ImmutableList.of(timeFrameName, op1, op2));
+ }
+
BigDecimal multiplier = BigDecimal.ONE;
BigDecimal divider = BigDecimal.ONE;
SqlTypeName sqlTypeName =
@@ -1829,19 +1952,22 @@ public class StandardConvertletTable extends ReflectiveConvertletTable {
case WEEK:
multiplier = BigDecimal.valueOf(DateTimeUtils.MILLIS_PER_SECOND);
divider = unit.multiplier;
- unit = TimeUnit.SECOND;
+ qualifier =
+ new SqlIntervalQualifier(
+ TimeUnit.SECOND, null, qualifier.getParserPosition());
break;
case QUARTER:
+ case CENTURY:
+ case MILLENNIUM:
divider = unit.multiplier;
- unit = TimeUnit.MONTH;
+ qualifier =
+ new SqlIntervalQualifier(
+ TimeUnit.MONTH, null, qualifier.getParserPosition());
break;
default:
+ qualifier = new SqlIntervalQualifier(unit, null, qualifier.getParserPosition());
break;
}
- final SqlIntervalQualifier qualifier =
- new SqlIntervalQualifier(unit, null, SqlParserPos.ZERO);
- final RexNode op2 = cx.convertExpression(call.operand(2));
- final RexNode op1 = cx.convertExpression(call.operand(1));
final RelDataType intervalType =
cx.getTypeFactory()
.createTypeWithNullability(
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/tools/RelBuilder.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/tools/RelBuilder.java
index 3401942c57f..0c21d0af78c 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/tools/RelBuilder.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/tools/RelBuilder.java
@@ -106,6 +106,7 @@ import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.type.TableFunctionReturnTypeInference;
import org.apache.calcite.sql.validate.SqlValidatorUtil;
import org.apache.calcite.sql2rel.SqlToRelConverter;
+import org.apache.calcite.util.DateString;
import org.apache.calcite.util.Holder;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.ImmutableIntList;
@@ -156,7 +157,7 @@ import static org.apache.calcite.util.Static.RESOURCE;
* <p>FLINK modifications are at lines
*
* <ol>
- * <li>Should be removed after fix of FLINK-29804: Lines 2945 ~ 2948
+ * <li>Should be removed after fix of FLINK-29804: Lines 3000 ~ 3003
* </ol>
*/
@Value.Enclosing
@@ -470,6 +471,8 @@ public class RelBuilder {
} else if (value instanceof Enum) {
return rexBuilder.makeLiteral(
value, getTypeFactory().createSqlType(SqlTypeName.SYMBOL));
+ } else if (value instanceof DateString) {
+ return rexBuilder.makeDateLiteral((DateString) value);
} else {
throw new IllegalArgumentException(
"cannot convert " + value + " (" + value.getClass() + ") to a constant");
@@ -1182,7 +1185,7 @@ public class RelBuilder {
*
* @see #project
*/
- public RexNode alias(RexNode expr, @Nullable String alias) {
+ public RexNode alias(RexNode expr, String alias) {
final RexNode aliasLiteral = literal(alias);
switch (expr.getKind()) {
case AS:
@@ -1988,7 +1991,25 @@ public class RelBuilder {
Iterable<? extends RexNode> nodes,
Iterable<? extends @Nullable String> fieldNames,
boolean force) {
- return project_(nodes, fieldNames, ImmutableList.of(), force);
+ return project(nodes, fieldNames, force, ImmutableSet.of());
+ }
+
+ /**
+ * The same with {@link #project(Iterable, Iterable, boolean)}, with additional variablesSet
+ * param.
+ *
+ * @param nodes Expressions
+ * @param fieldNames Suggested field names
+ * @param force create project even if it is identity
+ * @param variablesSet Correlating variables that are set when reading a row from the input, and
+ * which may be referenced from the projection expressions
+ */
+ public RelBuilder project(
+ Iterable<? extends RexNode> nodes,
+ Iterable<? extends @Nullable String> fieldNames,
+ boolean force,
+ Iterable<CorrelationId> variablesSet) {
+ return project_(nodes, fieldNames, ImmutableList.of(), force, variablesSet);
}
/** Creates a {@link Project} of all original fields, plus the given expressions. */
@@ -1997,7 +2018,7 @@ public class RelBuilder {
}
/** Creates a {@link Project} of all original fields, plus the given list of expressions. */
- public RelBuilder projectPlus(Iterable<RexNode> nodes) {
+ public RelBuilder projectPlus(Iterable<? extends RexNode> nodes) {
return project(Iterables.concat(fields(), nodes));
}
@@ -2061,10 +2082,12 @@ public class RelBuilder {
Iterable<? extends RexNode> nodes,
Iterable<? extends @Nullable String> fieldNames,
Iterable<RelHint> hints,
- boolean force) {
+ boolean force,
+ Iterable<CorrelationId> variablesSet) {
final Frame frame = requireNonNull(peek_(), "frame stack is empty");
final RelDataType inputRowType = frame.rel.getRowType();
final List<RexNode> nodeList = Lists.newArrayList(nodes);
+ final Set<CorrelationId> variables = ImmutableSet.copyOf(variablesSet);
// Perform a quick check for identity. We'll do a deeper check
// later when we've derived column names.
@@ -2077,8 +2100,9 @@ public class RelBuilder {
fieldNameList.add(null);
}
+ // Do not merge projection when top projection has correlation variables
bloat:
- if (frame.rel instanceof Project && config.bloat() >= 0) {
+ if (frame.rel instanceof Project && config.bloat() >= 0 && variables.isEmpty()) {
final Project project = (Project) frame.rel;
// Populate field names. If the upper expression is an input ref and does
// not have a recommended name, use the name of the underlying field.
@@ -2123,7 +2147,13 @@ public class RelBuilder {
final ImmutableSet.Builder<RelHint> mergedHints = ImmutableSet.builder();
mergedHints.addAll(project.getHints());
mergedHints.addAll(hints);
- return project_(newNodes, fieldNameList, mergedHints.build(), force);
+ // Keep bottom projection's variablesSet.
+ return project_(
+ newNodes,
+ fieldNameList,
+ mergedHints.build(),
+ force,
+ ImmutableSet.copyOf(project.getVariablesSet()));
}
// Simplify expressions.
@@ -2209,7 +2239,8 @@ public class RelBuilder {
frame.rel,
ImmutableList.copyOf(hints),
ImmutableList.copyOf(nodeList),
- fieldNameList);
+ fieldNameList,
+ variables);
stack.pop();
stack.push(new Frame(project, fields.build()));
return this;
@@ -2233,6 +2264,30 @@ public class RelBuilder {
Iterable<? extends RexNode> nodes,
@Nullable Iterable<? extends @Nullable String> fieldNames,
boolean force) {
+ return projectNamed(nodes, fieldNames, force, ImmutableSet.of());
+ }
+
+ /**
+ * Creates a {@link Project} of the given expressions and field names, and optionally
+ * optimizing.
+ *
+ * <p>If {@code fieldNames} is null, or if a particular entry in {@code fieldNames} is null,
+ * derives field names from the input expressions.
+ *
+ * <p>If {@code force} is false, and the input is a {@code Project}, and the expressions make
+ * the trivial projection ($0, $1, ...), modifies the input.
+ *
+ * @param nodes Expressions
+ * @param fieldNames Suggested field names, or null to generate
+ * @param force Whether to create a renaming Project if the projections are trivial
+ * @param variablesSet Correlating variables that are set when reading a row from the input, and
+ * which may be referenced from the projection expressions
+ */
+ public RelBuilder projectNamed(
+ Iterable<? extends RexNode> nodes,
+ @Nullable Iterable<? extends @Nullable String> fieldNames,
+ boolean force,
+ Iterable<CorrelationId> variablesSet) {
@SuppressWarnings("unchecked")
final List<? extends RexNode> nodeList =
nodes instanceof List ? (List) nodes : ImmutableList.copyOf(nodes);
@@ -2278,7 +2333,7 @@ public class RelBuilder {
stack.push(new Frame(newValues, frame.fields));
}
} else {
- project(nodeList, rowType.getFieldNames(), force);
+ project(nodeList, rowType.getFieldNames(), force, variablesSet);
}
return this;
}
@@ -3474,7 +3529,8 @@ public class RelBuilder {
sort,
project.getHints(),
project.getProjects(),
- Pair.right(project.getNamedProjects())));
+ Pair.right(project.getNamedProjects()),
+ project.getVariablesSet()));
return this;
}
}
diff --git a/flink-table/flink-table-planner/src/main/resources/META-INF/NOTICE b/flink-table/flink-table-planner/src/main/resources/META-INF/NOTICE
index 7ef6717283f..2918375f6cd 100644
--- a/flink-table/flink-table-planner/src/main/resources/META-INF/NOTICE
+++ b/flink-table/flink-table-planner/src/main/resources/META-INF/NOTICE
@@ -8,9 +8,10 @@ This project bundles the following dependencies under the Apache Software Licens
- com.google.guava:guava:32.1.3-jre
- com.google.guava:failureaccess:1.0.1
-- org.apache.calcite:calcite-core:1.32.0
-- org.apache.calcite:calcite-linq4j:1.32.0
-- org.apache.calcite.avatica:avatica-core:1.22.0
+- org.apache.calcite:calcite-core:1.33.0
+- org.apache.calcite:calcite-linq4j:1.33.0
+- org.apache.calcite.avatica:avatica-core:1.23.0
+- org.apache.commons:commons-math3:3.6.1
- commons-codec:commons-codec:1.15
- commons-io:commons-io:2.15.1
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/calcite/FlinkLogicalRelFactories.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/calcite/FlinkLogicalRelFactories.scala
index 21a8476b77b..a952e0b9ba6 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/calcite/FlinkLogicalRelFactories.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/calcite/FlinkLogicalRelFactories.scala
@@ -79,11 +79,12 @@ object FlinkLogicalRelFactories {
/** Implementation of [[ProjectFactory]] that returns a [[FlinkLogicalCalc]]. */
class ProjectFactoryImpl extends ProjectFactory {
- def createProject(
+ override def createProject(
input: RelNode,
hints: util.List[RelHint],
childExprs: util.List[_ <: RexNode],
- fieldNames: util.List[_ <: String]): RelNode = {
+ fieldNames: util.List[_ <: String],
+ variablesSet: util.Set[CorrelationId]): RelNode = {
val rexBuilder = input.getCluster.getRexBuilder
val inputRowType = input.getRowType
val programBuilder = new RexProgramBuilder(inputRowType, rexBuilder)
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashJoinTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashJoinTest.xml
index 16537dc550a..d5739c89524 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashJoinTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashJoinTest.xml
@@ -683,4 +683,23 @@ HashJoin(joinType=[LeftOuterJoin], where=[(k = k0)], select=[k, v, k0, v0], isBr
]]>
</Resource>
</TestCase>
+ <TestCase name="testRightOuterJoinOnFalse">
+ <Resource name="sql">
+ <![CDATA[SELECT * FROM MyTable2 RIGHT OUTER JOIN MyTable1 ON false]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalProject(d=[$0], e=[$1], f=[$2], g=[$3], h=[$4], a=[$5], b=[$6], c=[$7])
++- LogicalJoin(condition=[false], joinType=[right])
+ :- LogicalTableScan(table=[[default_catalog, default_database, MyTable2, source: [TestTableSource(d, e, f, g, h)]]])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable1, source: [TestTableSource(a, b, c)]]])
+]]>
+ </Resource>
+ <Resource name="optimized exec plan">
+ <![CDATA[
+Calc(select=[null:INTEGER AS d, null:BIGINT AS e, null:INTEGER AS f, null:VARCHAR(2147483647) AS g, null:BIGINT AS h, a, b, c])
++- LegacyTableSourceScan(table=[[default_catalog, default_database, MyTable1, source: [TestTableSource(a, b, c)]]], fields=[a, b, c])
+]]>
+ </Resource>
+ </TestCase>
</Root>
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/NestedLoopJoinTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/NestedLoopJoinTest.xml
index 770853f2fbe..9582daba656 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/NestedLoopJoinTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/NestedLoopJoinTest.xml
@@ -917,9 +917,7 @@ LogicalProject(d=[$0], e=[$1], f=[$2], g=[$3], h=[$4], a=[$5], b=[$6], c=[$7])
</Resource>
<Resource name="optimized exec plan">
<![CDATA[
-NestedLoopJoin(joinType=[RightOuterJoin], where=[true], select=[d, e, f, g, h, a, b, c], build=[left])
-:- Exchange(distribution=[broadcast])
-: +- Values(tuples=[[]], values=[d, e, f, g, h])
+Calc(select=[null:INTEGER AS d, null:BIGINT AS e, null:INTEGER AS f, null:VARCHAR(2147483647) AS g, null:BIGINT AS h, a, b, c])
+- LegacyTableSourceScan(table=[[default_catalog, default_database, MyTable1, source: [TestTableSource(a, b, c)]]], fields=[a, b, c])
]]>
</Resource>
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashJoinTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashJoinTest.xml
index 1b14167bace..f88db3febe0 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashJoinTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashJoinTest.xml
@@ -795,4 +795,23 @@ HashJoin(joinType=[LeftOuterJoin], where=[(k = k0)], select=[k, v, k0, v0], buil
]]>
</Resource>
</TestCase>
+ <TestCase name="testRightOuterJoinOnFalse">
+ <Resource name="sql">
+ <![CDATA[SELECT * FROM MyTable2 RIGHT OUTER JOIN MyTable1 ON false]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalProject(d=[$0], e=[$1], f=[$2], g=[$3], h=[$4], a=[$5], b=[$6], c=[$7])
++- LogicalJoin(condition=[false], joinType=[right])
+ :- LogicalTableScan(table=[[default_catalog, default_database, MyTable2, source: [TestTableSource(d, e, f, g, h)]]])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable1, source: [TestTableSource(a, b, c)]]])
+]]>
+ </Resource>
+ <Resource name="optimized exec plan">
+ <![CDATA[
+Calc(select=[null:INTEGER AS d, null:BIGINT AS e, null:INTEGER AS f, null:VARCHAR(2147483647) AS g, null:BIGINT AS h, a, b, c])
++- LegacyTableSourceScan(table=[[default_catalog, default_database, MyTable1, source: [TestTableSource(a, b, c)]]], fields=[a, b, c])
+]]>
+ </Resource>
+ </TestCase>
</Root>
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeJoinTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeJoinTest.xml
index 855fd71c969..c6dfb533767 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeJoinTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeJoinTest.xml
@@ -795,4 +795,23 @@ SortMergeJoin(joinType=[LeftOuterJoin], where=[(k = k0)], select=[k, v, k0, v0])
]]>
</Resource>
</TestCase>
+ <TestCase name="testRightOuterJoinOnFalse">
+ <Resource name="sql">
+ <![CDATA[SELECT * FROM MyTable2 RIGHT OUTER JOIN MyTable1 ON false]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalProject(d=[$0], e=[$1], f=[$2], g=[$3], h=[$4], a=[$5], b=[$6], c=[$7])
++- LogicalJoin(condition=[false], joinType=[right])
+ :- LogicalTableScan(table=[[default_catalog, default_database, MyTable2, source: [TestTableSource(d, e, f, g, h)]]])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable1, source: [TestTableSource(a, b, c)]]])
+]]>
+ </Resource>
+ <Resource name="optimized exec plan">
+ <![CDATA[
+Calc(select=[null:INTEGER AS d, null:BIGINT AS e, null:INTEGER AS f, null:VARCHAR(2147483647) AS g, null:BIGINT AS h, a, b, c])
++- LegacyTableSourceScan(table=[[default_catalog, default_database, MyTable1, source: [TestTableSource(a, b, c)]]], fields=[a, b, c])
+]]>
+ </Resource>
+ </TestCase>
</Root>
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/JoinDependentConditionDerivationRuleTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/JoinDependentConditionDerivationRuleTest.xml
index 5dfeaddc35c..be18f2318c9 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/JoinDependentConditionDerivationRuleTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/JoinDependentConditionDerivationRuleTest.xml
@@ -106,12 +106,12 @@ LogicalProject(a=[$0], d=[$3])
</TestCase>
<TestCase name="testAndOr">
<Resource name="sql">
- <![CDATA[SELECT a, d FROM MyTable1, MyTable2 WHERE ((a = 1 AND d = 2) OR (a = 2 AND d = 1)) AND ((a = 3 AND d = 4) OR (a = 4 AND d = 3))]]>
+ <![CDATA[SELECT a, d FROM MyTable1, MyTable2 WHERE ((a = 1 AND d = 2) OR (a = 2 AND d = 1))]]>
</Resource>
<Resource name="ast">
<![CDATA[
LogicalProject(a=[$0], d=[$3])
-+- LogicalFilter(condition=[AND(OR(AND(=($0, 1), =($3, 2)), AND(=($0, 2), =($3, 1))), OR(AND(=($0, 3), =($3, 4)), AND(=($0, 4), =($3, 3))))])
++- LogicalFilter(condition=[OR(AND(=($0, 1), =($3, 2)), AND(=($0, 2), =($3, 1)))])
+- LogicalJoin(condition=[true], joinType=[inner])
:- LogicalTableScan(table=[[default_catalog, default_database, MyTable1, source: [TestTableSource(a, b, c)]]])
+- LogicalTableScan(table=[[default_catalog, default_database, MyTable2, source: [TestTableSource(d, e, f, g, h)]]])
@@ -120,7 +120,7 @@ LogicalProject(a=[$0], d=[$3])
<Resource name="optimized rel plan">
<![CDATA[
LogicalProject(a=[$0], d=[$3])
-+- LogicalJoin(condition=[AND(OR(AND(=($0, 1), =($3, 2)), AND(=($0, 2), =($3, 1))), OR(AND(=($0, 3), =($3, 4)), AND(=($0, 4), =($3, 3))), SEARCH($0, Sarg[1, 2]), SEARCH($3, Sarg[1, 2]), SEARCH($0, Sarg[3, 4]), SEARCH($3, Sarg[3, 4]))], joinType=[inner])
++- LogicalJoin(condition=[AND(OR(AND(=($0, 1), =($3, 2)), AND(=($0, 2), =($3, 1))), SEARCH($0, Sarg[1, 2]), SEARCH($3, Sarg[1, 2]))], joinType=[inner])
:- LogicalTableScan(table=[[default_catalog, default_database, MyTable1, source: [TestTableSource(a, b, c)]]])
+- LogicalTableScan(table=[[default_catalog, default_database, MyTable2, source: [TestTableSource(d, e, f, g, h)]]])
]]>
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/RemoveSingleAggregateRuleTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/RemoveSingleAggregateRuleTest.xml
index d8afcc2b74a..a864eaaeef1 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/RemoveSingleAggregateRuleTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/RemoveSingleAggregateRuleTest.xml
@@ -22,7 +22,7 @@ limitations under the License.
</Resource>
<Resource name="ast">
<![CDATA[
-LogicalProject(EXPR$0=[$SCALAR_QUERY({
+LogicalProject(variablesSet=[[$cor0]], EXPR$0=[$SCALAR_QUERY({
LogicalProject(EXPR$0=[-($0, 1)])
LogicalAggregate(group=[{}], agg#0=[COUNT($0)])
LogicalProject(x=[$0])
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubqueryCorrelateVariablesValidationTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubqueryCorrelateVariablesValidationTest.xml
index 6a8db57acf8..9c42ceef11f 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubqueryCorrelateVariablesValidationTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubqueryCorrelateVariablesValidationTest.xml
@@ -26,7 +26,7 @@ WHERE t1a = 'test'
</Resource>
<Resource name="ast">
<![CDATA[
-LogicalProject(min_t1d=[$SCALAR_QUERY({
+LogicalProject(variablesSet=[[$cor0]], min_t1d=[$SCALAR_QUERY({
LogicalAggregate(group=[{}], EXPR$0=[MIN($0)])
LogicalProject($f0=[$cor0.t1d])
LogicalFilter(condition=[=($0, _UTF-16LE'test')])
@@ -65,7 +65,7 @@ FROM t1
</Resource>
<Resource name="ast">
<![CDATA[
-LogicalProject(min_t3d=[$SCALAR_QUERY({
+LogicalProject(variablesSet=[[$cor0]], min_t3d=[$SCALAR_QUERY({
LogicalAggregate(group=[{}], EXPR$0=[MIN($0)])
LogicalProject(t3d=[$3])
LogicalFilter(condition=[=($0, $cor0.t1a)])
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/expressions/TemporalTypesTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/expressions/TemporalTypesTest.scala
index 5011495ec80..4fba6e34a08 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/expressions/TemporalTypesTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/expressions/TemporalTypesTest.scala
@@ -1448,7 +1448,7 @@ class TemporalTypesTest extends ExpressionTestBase {
testExpectedSqlException(
s"TIMESTAMPDIFF(SECOND, ${timestampLtz("1970-01-01 00:00:00.123")}, 'test_string_type')",
"Cannot apply 'TIMESTAMPDIFF' to arguments of type" +
- " 'TIMESTAMPDIFF(<SYMBOL>, <TIMESTAMP_WITH_LOCAL_TIME_ZONE(3)>, <CHAR(16)>)'." +
+ " 'TIMESTAMPDIFF(<INTERVAL SECOND>, <TIMESTAMP_WITH_LOCAL_TIME_ZONE(3)>, <CHAR(16)>)'." +
" Supported form(s): 'TIMESTAMPDIFF(<ANY>, <DATETIME>, <DATETIME>)'"
)
}
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashJoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashJoinTest.scala
index 0f3c494ee5e..0d608b04b31 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashJoinTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashJoinTest.scala
@@ -69,13 +69,6 @@ class BroadcastHashJoinTest extends JoinTestBase {
.isInstanceOf[TableException]
}
- @Test
- override def testRightOuterJoinOnFalse(): Unit = {
- assertThatThrownBy(() => super.testRightOuterJoinOnFalse())
- .hasMessageContaining("Cannot generate a valid execution plan for the given query")
- .isInstanceOf[TableException]
- }
-
@Test
override def testRightOuterJoinWithNonEquiPred(): Unit = {
assertThatThrownBy(() => super.testRightOuterJoinWithNonEquiPred())
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashJoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashJoinTest.scala
index 36687fb6fd9..c9ad5909f9b 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashJoinTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashJoinTest.scala
@@ -69,13 +69,6 @@ class ShuffledHashJoinTest extends JoinTestBase {
.isInstanceOf[TableException]
}
- @Test
- override def testRightOuterJoinOnFalse(): Unit = {
- assertThatThrownBy(() => super.testRightOuterJoinOnFalse())
- .hasMessageContaining("Cannot generate a valid execution plan for the given query")
- .isInstanceOf[TableException]
- }
-
@Test
override def testRightOuterJoinWithNonEquiPred(): Unit = {
assertThatThrownBy(() => super.testRightOuterJoinWithNonEquiPred())
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeJoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeJoinTest.scala
index 2d7cf24fab4..7e526138e26 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeJoinTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeJoinTest.scala
@@ -66,13 +66,6 @@ class SortMergeJoinTest extends JoinTestBase {
.isInstanceOf[TableException]
}
- @Test
- override def testRightOuterJoinOnFalse(): Unit = {
- assertThatThrownBy(() => super.testRightOuterJoinOnFalse())
- .hasMessageContaining("Cannot generate a valid execution plan for the given query")
- .isInstanceOf[TableException]
- }
-
@Test
override def testRightOuterJoinWithNonEquiPred(): Unit = {
assertThatThrownBy(() => super.testRightOuterJoinWithNonEquiPred())
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdDistinctRowCountTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdDistinctRowCountTest.scala
index 9ad9a6de76e..568307603f8 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdDistinctRowCountTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdDistinctRowCountTest.scala
@@ -613,7 +613,7 @@ class FlinkRelMdDistinctRowCountTest extends FlinkRelMdHandlerTestBase {
2.0e7,
mq.getDistinctRowCount(logicalInnerJoinNotOnUniqueKeys, ImmutableBitSet.of(0), null))
assertEquals(
- 5.0569644545e8,
+ 5.0569644724678594e8,
mq.getDistinctRowCount(logicalInnerJoinNotOnUniqueKeys, ImmutableBitSet.of(1), null),
1e-2)
@@ -621,14 +621,14 @@ class FlinkRelMdDistinctRowCountTest extends FlinkRelMdHandlerTestBase {
2.0e7,
mq.getDistinctRowCount(logicalLeftJoinOnUniqueKeys, ImmutableBitSet.of(0), null))
assertEquals(
- 5.0569644545e8,
+ 5.0569644724678594e8,
mq.getDistinctRowCount(logicalLeftJoinOnUniqueKeys, ImmutableBitSet.of(1), null),
1e-2)
assertEquals(
2.0e7,
mq.getDistinctRowCount(logicalLeftJoinNotOnUniqueKeys, ImmutableBitSet.of(0), null))
assertEquals(
- 5.0569644545e8,
+ 5.0569644724678594e8,
mq.getDistinctRowCount(logicalLeftJoinNotOnUniqueKeys, ImmutableBitSet.of(1), null),
1e-2)
@@ -644,7 +644,7 @@ class FlinkRelMdDistinctRowCountTest extends FlinkRelMdHandlerTestBase {
2.0e7,
mq.getDistinctRowCount(logicalRightJoinNotOnUniqueKeys, ImmutableBitSet.of(0), null))
assertEquals(
- 5.0569644545e8,
+ 5.0569644724678594e8,
mq.getDistinctRowCount(logicalRightJoinNotOnUniqueKeys, ImmutableBitSet.of(1), null),
1e-2)
@@ -652,14 +652,14 @@ class FlinkRelMdDistinctRowCountTest extends FlinkRelMdHandlerTestBase {
2.0e7,
mq.getDistinctRowCount(logicalFullJoinOnUniqueKeys, ImmutableBitSet.of(0), null))
assertEquals(
- 5.0569644545e8,
+ 5.0569644724678594e8,
mq.getDistinctRowCount(logicalFullJoinOnUniqueKeys, ImmutableBitSet.of(1), null),
1e-2)
assertEquals(
2.0e7,
mq.getDistinctRowCount(logicalFullJoinNotOnUniqueKeys, ImmutableBitSet.of(0), null))
assertEquals(
- 5.0569644545e8,
+ 5.0569644724678594e8,
mq.getDistinctRowCount(logicalFullJoinNotOnUniqueKeys, ImmutableBitSet.of(1), null),
1e-2)
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdPopulationSizeTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdPopulationSizeTest.scala
index 9c9925b8348..8de36795609 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdPopulationSizeTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdPopulationSizeTest.scala
@@ -334,15 +334,15 @@ class FlinkRelMdPopulationSizeTest extends FlinkRelMdHandlerTestBase {
assertEquals(1.0, mq.getPopulationSize(logicalLeftJoinNotOnUniqueKeys, ImmutableBitSet.of()))
assertEquals(2.0e7, mq.getPopulationSize(logicalLeftJoinNotOnUniqueKeys, ImmutableBitSet.of(0)))
assertEquals(
- 5.0569644545e8,
+ 5.0569644724678594e8,
mq.getPopulationSize(logicalLeftJoinNotOnUniqueKeys, ImmutableBitSet.of(1)),
1e-2)
assertEquals(
- 8.0e8,
+ 7.999999791508117e8,
mq.getPopulationSize(logicalLeftJoinNotOnUniqueKeys, ImmutableBitSet.of(1, 5)),
1e-2)
assertEquals(
- 7.9377199253e8,
+ 7.937727457918736e8,
mq.getPopulationSize(logicalLeftJoinNotOnUniqueKeys, ImmutableBitSet.of(0, 6)),
1e-2)
@@ -352,15 +352,15 @@ class FlinkRelMdPopulationSizeTest extends FlinkRelMdHandlerTestBase {
mq.getPopulationSize(logicalRightJoinOnLHSUniqueKeys, ImmutableBitSet.of(0)),
1e-2)
assertEquals(
- 1.975207027e7,
+ 1.9752070389525224e7,
mq.getPopulationSize(logicalRightJoinOnLHSUniqueKeys, ImmutableBitSet.of(1)),
1e-2)
assertEquals(
- 2.0e7,
+ 1.999999987845058e7,
mq.getPopulationSize(logicalRightJoinOnLHSUniqueKeys, ImmutableBitSet.of(1, 5)),
1e-2)
assertEquals(
- 1.999606902e7,
+ 1.9996088147299763e7,
mq.getPopulationSize(logicalRightJoinOnLHSUniqueKeys, ImmutableBitSet.of(0, 6)),
1e-2)
@@ -368,7 +368,7 @@ class FlinkRelMdPopulationSizeTest extends FlinkRelMdHandlerTestBase {
assertEquals(2.0e7, mq.getPopulationSize(logicalFullJoinWithoutEquiCond, ImmutableBitSet.of(0)))
assertEquals(8.0e8, mq.getPopulationSize(logicalFullJoinWithoutEquiCond, ImmutableBitSet.of(1)))
assertEquals(
- 8.0e15,
+ 6.295509444597865e15,
mq.getPopulationSize(logicalFullJoinWithoutEquiCond, ImmutableBitSet.of(1, 5)))
assertEquals(
5.112e10,
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinDependentConditionDerivationRuleTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinDependentConditionDerivationRuleTest.scala
index 91cfdc9dfa0..7d2d12f9462 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinDependentConditionDerivationRuleTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinDependentConditionDerivationRuleTest.scala
@@ -83,7 +83,7 @@ class JoinDependentConditionDerivationRuleTest extends TableTestBase {
@Test
def testAndOr(): Unit = {
val sqlQuery = "SELECT a, d FROM MyTable1, MyTable2 WHERE " +
- "((a = 1 AND d = 2) OR (a = 2 AND d = 1)) AND ((a = 3 AND d = 4) OR (a = 4 AND d = 3))"
+ "((a = 1 AND d = 2) OR (a = 2 AND d = 1))"
util.verifyRelPlan(sqlQuery)
}
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/AggregateValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/AggregateValidationTest.scala
index b3b76bb3a82..476ac673aec 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/AggregateValidationTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/AggregateValidationTest.scala
@@ -129,7 +129,7 @@ class AggregateValidationTest extends TableTestBase {
// If there are two parameters, second one must be character literal.
expectExceptionThrown(
"SELECT listagg(c, d) FROM T GROUP BY a",
- "Supported form(s): 'LISTAGG(<CHARACTER>)'\n'LISTAGG(<CHARACTER>, <CHARACTER_LITERAL>)",
+ "Argument to function 'LISTAGG' must be a literal",
classOf[ValidationException]
)
}
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/utils/FlinkRelMdUtilTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/utils/FlinkRelMdUtilTest.scala
index f8fc56d48d8..bfd1a8098c7 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/utils/FlinkRelMdUtilTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/utils/FlinkRelMdUtilTest.scala
@@ -31,18 +31,11 @@ class FlinkRelMdUtilTest {
Assertions.assertEquals(
BigDecimal(0.31606027941427883),
BigDecimal.valueOf(FlinkRelMdUtil.numDistinctVals(0.5, 0.5)))
-
- // This case should be removed once CALCITE-4351 is fixed.
- Assertions.assertEquals(Double.NaN, RelMdUtil.numDistinctVals(0.5, 0.5))
}
@Test
def testNumDistinctValsWithLargeInputs(): Unit = {
Assertions.assertNotEquals(0.0, FlinkRelMdUtil.numDistinctVals(1e18, 1e10))
Assertions.assertEquals(9.99999993922529e9, FlinkRelMdUtil.numDistinctVals(1e18, 1e10), 1d)
- // this test will fail once CALCITE-4351 is fixed
- // in that case FlinkRelMdUtil#numDistinctVals should be removed
- // see FLINK-19780
- Assertions.assertEquals(0.0, RelMdUtil.numDistinctVals(1e18, 1e10))
}
}
diff --git a/flink-table/pom.xml b/flink-table/pom.xml
index d8f0fff4aa6..4329be9be75 100644
--- a/flink-table/pom.xml
+++ b/flink-table/pom.xml
@@ -78,8 +78,8 @@ under the License.
</dependencyManagement>
<properties>
- <calcite.version>1.32.0</calcite.version>
- <!-- Calcite 1.32.0 depends on 3.1.8,
+ <calcite.version>1.33.0</calcite.version>
+ <!-- Calcite 1.33.0 depends on 3.1.8,
at the same time minimum 3.1.x Janino version passing Flink tests without WAs is 3.1.10,
more details are in FLINK-27995 -->
<janino.version>3.1.10</janino.version>