You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by sn...@apache.org on 2026/07/06 18:47:39 UTC
(flink) branch master updated (9661ab549bc -> c73b956225a)
This is an automated email from the ASF dual-hosted git repository.
snuyanzin pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from 9661ab549bc [FLINK-36753][runtime]Adaptive Scheduler actively triggers a Checkpoint after all resources are ready
new fb9c511f590 [FLINK-39859][table] Replace `Javaconverions` with `JavaConverters` in `RemoteCalcSplitRule` and `RemoteCalcSpliRule`
new 2b063e5d220 [FLINK-39859][table] Remove deprecated `JavaConversions` from `FlinkStatistic`
new 4c33041e67f [FLINK-39859][table] Remove deprecated `JavaConversions` from `Sink` and its children
new 074f0f98444 [FLINK-39859][table] Remove deprecated `JavaConversions` from `FlinkRelDistribution`
new c09b543ea6d [FLINK-39859][table] Remove deprecated `JavaConversions` from `FlinkRelMdUtil`
new 66ca7c54a6e [FLINK-39859][table] Remove usage of deprecated `getIndicatorCount`
new f1c2318be9c [FLINK-39859][table] Replace deprecated usage of `SqlLiteral#createTimestamp`
new c73b956225a [FLINK-39859][table] Upgrade Calcite to 1.41.0
The 8 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
flink-table/flink-sql-parser/pom.xml | 13 +-
.../src/main/codegen/data/Parser.tdd | 1 +
.../src/main/codegen/templates/Parser.jj | 60 +-
.../org/apache/calcite/sql/SqlMapTypeNameSpec.java | 9 +-
.../sql/parser/validate/FlinkSqlConformance.java | 10 +
flink-table/flink-table-calcite-bridge/pom.xml | 10 +-
flink-table/flink-table-planner/pom.xml | 13 +
.../calcite/rel/rules/SubQueryRemoveRule.java | 150 +-
.../calcite/rel/type/RelDataTypeFactoryImpl.java | 16 +-
.../java/org/apache/calcite/rex/RexSimplify.java | 3715 ++++++++++++++++++++
.../main/java/org/apache/calcite/rex/RexUtil.java | 134 +-
.../org/apache/calcite/runtime/SqlFunctions.java | 502 ++-
.../calcite/sql/fun/SqlCoalesceFunction.java | 9 +-
.../calcite/sql/fun/SqlStdOperatorTable.java | 79 +-
.../calcite/sql/type/SqlTypeFactoryImpl.java | 6 +-
.../org/apache/calcite/sql/type/SqlTypeUtil.java | 219 +-
.../calcite/sql/validate/SqlValidatorImpl.java | 395 ++-
.../apache/calcite/sql2rel/RelDecorrelator.java | 333 +-
.../apache/calcite/sql2rel/SqlToRelConverter.java | 266 +-
.../planner/calcite/FlinkCalciteSqlValidator.java | 2 +
.../plan/nodes/exec/common/CommonExecMatch.java | 7 +
.../logical/FlinkAggregateJoinTransposeRule.java | 2 +-
...rrelateToJoinFromTemporalTableFunctionRule.java | 6 +
.../rules/logical/RemoteCorrelateSplitRule.java | 6 +
.../SplitPythonConditionFromCorrelateRule.java | 6 +
.../physical/common/CommonPhysicalMatchRule.java | 6 +
.../flink/table/planner/plan/utils/AsyncUtil.java | 6 +
.../planner/plan/utils/ConstantFoldingUtil.java | 6 +
.../flink/table/planner/plan/utils/PythonUtil.java | 11 +
.../src/main/resources/META-INF/NOTICE | 8 +-
.../table/planner/codegen/ExprCodeGenerator.scala | 5 +
.../planner/plan/nodes/calcite/LogicalSink.scala | 6 +-
.../table/planner/plan/nodes/calcite/Sink.scala | 4 +-
.../plan/nodes/logical/FlinkLogicalSink.scala | 4 +-
.../nodes/physical/stream/StreamPhysicalSink.scala | 6 +-
.../plan/rules/logical/RemoteCalcSplitRule.scala | 54 +-
.../table/planner/plan/stats/FlinkStatistic.scala | 4 +-
.../planner/plan/trait/FlinkRelDistribution.scala | 12 +-
.../table/planner/plan/utils/FlinkRelMdUtil.scala | 44 +-
.../flink/table/planner/plan/utils/MatchUtil.scala | 7 +-
.../planner/plan/utils/RexNodeExtractor.scala | 61 +-
.../table/planner/plan/batch/sql/ValuesTest.xml | 6 +-
.../plan/batch/sql/agg/GroupingSetsTest.xml | 22 +-
.../plan/batch/sql/join/BroadcastHashJoinTest.xml | 2 +-
.../plan/batch/sql/join/NestedLoopJoinTest.xml | 2 +-
.../plan/batch/sql/join/ShuffledHashJoinTest.xml | 2 +-
.../plan/batch/sql/join/SortMergeJoinTest.xml | 2 +-
.../logical/DecomposeGroupingSetsRuleTest.xml | 13 +-
.../plan/rules/logical/FlinkFilterJoinRuleTest.xml | 4 +-
.../logical/RemoveSingleAggregateRuleTest.xml | 18 +-
.../logical/SimplifyFilterConditionRuleTest.xml | 4 +-
.../logical/SimplifyJoinConditionRuleTest.xml | 20 +-
.../logical/subquery/SubQueryAntiJoinTest.xml | 99 +-
.../logical/subquery/SubQuerySemiJoinTest.xml | 42 +-
.../plan/stream/sql/InsertIntoValuesTest.xml | 4 +-
.../table/planner/plan/stream/sql/ValuesTest.xml | 6 +-
.../plan/stream/sql/agg/GroupingSetsTest.xml | 36 +
.../metadata/FlinkRelMdDistinctRowCountTest.scala | 19 +-
.../metadata/FlinkRelMdPopulationSizeTest.scala | 4 +-
.../plan/stream/sql/agg/GroupingSetsTest.scala | 6 +-
.../runtime/batch/sql/agg/GroupingSetsITCase.scala | 7 +-
flink-table/pom.xml | 2 +-
62 files changed, 5951 insertions(+), 582 deletions(-)
create mode 100644 flink-table/flink-table-planner/src/main/java/org/apache/calcite/rex/RexSimplify.java
(flink) 03/08: [FLINK-39859][table] Remove deprecated `JavaConversions` from `Sink` and its children
Posted by sn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
snuyanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 4c33041e67f19a1a6c58cf579e7b7be21e45fdd9
Author: Sergey Nuyanzin <sn...@gmail.com>
AuthorDate: Sat Jul 4 08:38:04 2026 +0200
[FLINK-39859][table] Remove deprecated `JavaConversions` from `Sink` and its children
---
.../apache/flink/table/planner/plan/nodes/calcite/LogicalSink.scala | 6 +++---
.../org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala | 4 ++--
.../flink/table/planner/plan/nodes/logical/FlinkLogicalSink.scala | 4 ++--
.../planner/plan/nodes/physical/stream/StreamPhysicalSink.scala | 6 +++---
4 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/LogicalSink.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/LogicalSink.scala
index 22414924d7e..8915559360c 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/LogicalSink.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/LogicalSink.scala
@@ -28,7 +28,7 @@ import org.apache.calcite.rel.hint.RelHint
import java.util
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
/**
* Sub-class of [[Sink]] that is a relational expression which writes out data of input node into a
@@ -59,7 +59,7 @@ final class LogicalSink(
new LogicalSink(
cluster,
traitSet,
- inputs.head,
+ inputs.asScala.head,
hints,
contextResolvedTable,
tableSink,
@@ -90,7 +90,7 @@ object LogicalSink {
contextResolvedTable,
tableSink,
targetColumns,
- staticPartitions.toMap,
+ staticPartitions.asScala.toMap,
abilitySpecs,
conflictStrategy)
}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala
index 8279be977ca..2924cf9b76b 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/Sink.scala
@@ -30,7 +30,7 @@ import org.apache.calcite.rel.hint.RelHint
import java.util
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
/**
* Relational expression that writes out data of input node into a [[DynamicTableSink]].
@@ -82,7 +82,7 @@ abstract class Sink(
.mkString(","),
targetColumns != null
)
- .item("fields", getRowType.getFieldNames.mkString(", "))
+ .item("fields", getRowType.getFieldNames.asScala.mkString(", "))
.itemIf("hints", RelExplainUtil.hintsToString(hints), !hints.isEmpty)
}
}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalSink.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalSink.scala
index e176dd232d2..84e224fdd39 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalSink.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalSink.scala
@@ -32,7 +32,7 @@ import org.apache.calcite.rel.hint.RelHint
import java.util
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
/**
* Sub-class of [[Sink]] that is a relational expression which writes out data of input node into a
@@ -64,7 +64,7 @@ class FlinkLogicalSink(
new FlinkLogicalSink(
cluster,
traitSet,
- inputs.head,
+ inputs.asScala.head,
hints,
contextResolvedTable,
tableSink,
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalSink.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalSink.scala
index cf2a7d53c9e..0433a1a5474 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalSink.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalSink.scala
@@ -39,7 +39,7 @@ import org.apache.calcite.util.ImmutableBitSet
import java.util
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
/**
* Stream physical RelNode to write data into an external sink defined by a [[DynamicTableSink]].
@@ -148,7 +148,7 @@ class StreamPhysicalSink(
val pks = ImmutableBitSet.of(primaryKeys: _*)
val fmq = FlinkRelMetadataQuery.reuseOrCreate(getCluster.getMetadataQuery)
val changeLogUpsertKeys = fmq.getUpsertKeys(getInput)
- changeLogUpsertKeys != null && changeLogUpsertKeys.exists(pks.contains)
+ changeLogUpsertKeys != null && changeLogUpsertKeys.asScala.exists(pks.contains)
}
def getUpsertKeyNames: String = {
@@ -158,7 +158,7 @@ class StreamPhysicalSink(
"none"
} else {
val fieldNames = contextResolvedTable.getResolvedSchema.getColumnNames
- changeLogUpsertKeys
+ changeLogUpsertKeys.asScala
.map(uk => uk.toArray.map(fieldNames.get).mkString("[", ", ", "]"))
.mkString(", ")
}
(flink) 02/08: [FLINK-39859][table] Remove deprecated `JavaConversions` from `FlinkStatistic`
Posted by sn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
snuyanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 2b063e5d220ddce994a7954671b5671ea33fbd42
Author: Sergey Nuyanzin <sn...@gmail.com>
AuthorDate: Sat Jul 4 08:30:33 2026 +0200
[FLINK-39859][table] Remove deprecated `JavaConversions` from `FlinkStatistic`
---
.../org/apache/flink/table/planner/plan/stats/FlinkStatistic.scala | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/stats/FlinkStatistic.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/stats/FlinkStatistic.scala
index 7158a8039c9..077ccfd7583 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/stats/FlinkStatistic.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/stats/FlinkStatistic.scala
@@ -30,8 +30,6 @@ import javax.annotation.Nullable
import java.util
-import scala.collection.JavaConversions._
-
/** The class provides statistics for a [[org.apache.calcite.schema.Table]]. */
class FlinkStatistic(
tableStats: TableStats,
@@ -42,7 +40,7 @@ class FlinkStatistic(
require(tableStats != null, "tableStats should not be null")
require(
- uniqueKeys == null || !uniqueKeys.exists(keys => keys == null || keys.isEmpty),
+ uniqueKeys == null || !uniqueKeys.stream().anyMatch(keys => keys == null || keys.isEmpty),
"uniqueKeys contains invalid elements!")
/**
(flink) 06/08: [FLINK-39859][table] Remove usage of deprecated `getIndicatorCount`
Posted by sn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
snuyanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 66ca7c54a6e0ce53ba4177e127997c1979deae48
Author: Sergey Nuyanzin <sn...@gmail.com>
AuthorDate: Sat Jul 4 09:45:29 2026 +0200
[FLINK-39859][table] Remove usage of deprecated `getIndicatorCount`
---
.../planner/plan/rules/logical/FlinkAggregateJoinTransposeRule.java | 2 +-
.../org/apache/flink/table/planner/plan/utils/FlinkRelMdUtil.scala | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateJoinTransposeRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateJoinTransposeRule.java
index cbb6f44de91..75d8b5a3154 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateJoinTransposeRule.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateJoinTransposeRule.java
@@ -403,7 +403,7 @@ public class FlinkAggregateJoinTransposeRule extends RelOptRule {
// Aggregate above to sum up the sub-totals
final List<AggregateCall> newAggCalls = new ArrayList<>();
- final int groupIndicatorCount = aggregate.getGroupCount() + aggregate.getIndicatorCount();
+ final int groupIndicatorCount = aggregate.getGroupCount();
final int newLeftWidth = sides.get(0).newInput.getRowType().getFieldCount();
final List<RexNode> projects =
new ArrayList<>(rexBuilder.identityProjects(relBuilder.peek().getRowType()));
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/FlinkRelMdUtil.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/FlinkRelMdUtil.scala
index c2d4712c24c..c28ed3f48a8 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/FlinkRelMdUtil.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/FlinkRelMdUtil.scala
@@ -326,7 +326,7 @@ object FlinkRelMdUtil {
childKeyBuilder.set(fullGroupSet(bit))
} else if (!containsAllAggGroupKeys) {
// getIndicatorCount return 0 if auxGroupSet is not empty
- val agg = otherAggCalls.asJava.get(bit - (fullGroupSet.length + aggRel.getIndicatorCount))
+ val agg = otherAggCalls.asJava.get(bit - fullGroupSet.length)
aggCalls += agg
})
(childKeyBuilder.build(), aggCalls.toArray)
(flink) 05/08: [FLINK-39859][table] Remove deprecated `JavaConversions` from `FlinkRelMdUtil`
Posted by sn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
snuyanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit c09b543ea6d8886e4aae87ad712d03522708a4ae
Author: Sergey Nuyanzin <sn...@gmail.com>
AuthorDate: Sat Jul 4 09:05:57 2026 +0200
[FLINK-39859][table] Remove deprecated `JavaConversions` from `FlinkRelMdUtil`
---
.../table/planner/plan/utils/FlinkRelMdUtil.scala | 44 ++++++++++++----------
1 file changed, 24 insertions(+), 20 deletions(-)
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/FlinkRelMdUtil.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/FlinkRelMdUtil.scala
index 3e07a13ff91..c2d4712c24c 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/FlinkRelMdUtil.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/FlinkRelMdUtil.scala
@@ -37,12 +37,11 @@ import org.apache.calcite.rex._
import org.apache.calcite.sql.`type`.SqlTypeName.{TIME, TIMESTAMP}
import org.apache.calcite.sql.SqlKind
import org.apache.calcite.util.{ImmutableBitSet, NumberUtil}
-import org.apache.calcite.util.NumberUtil.multiply
import java.math.BigDecimal
import java.util
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable
/** FlinkRelMdUtil provides utility methods used by the metadata provider methods. */
@@ -176,7 +175,11 @@ object FlinkRelMdUtil {
*/
def makeNamePropertiesSelectivityRexNode(winAgg: WindowAggregate, predicate: RexNode): RexNode = {
val fullGroupSet = AggregateUtil.checkAndGetFullGroupSet(winAgg)
- makeNamePropertiesSelectivityRexNode(winAgg, fullGroupSet, winAgg.getNamedProperties, predicate)
+ makeNamePropertiesSelectivityRexNode(
+ winAgg,
+ fullGroupSet,
+ winAgg.getNamedProperties.asScala,
+ predicate)
}
/**
@@ -237,7 +240,7 @@ object FlinkRelMdUtil {
predicate,
pushable,
notPushable)
- if (notPushable.nonEmpty) {
+ if (notPushable.asScala.nonEmpty) {
val pred = RexUtil.composeConjunction(rexBuilder, notPushable, true)
val selectivity = RelMdUtil.guessSelectivity(pred)
val fun = rexBuilder.makeCall(
@@ -317,13 +320,13 @@ object FlinkRelMdUtil {
// does not need to take keys in aggregate call into consideration if groupKey contains all
// groupSet element in aggregate
val containsAllAggGroupKeys = fullGroupSet.indices.forall(groupKey.get)
- groupKey.foreach(
+ groupKey.asScala.foreach(
bit =>
if (bit < fullGroupSet.length) {
childKeyBuilder.set(fullGroupSet(bit))
} else if (!containsAllAggGroupKeys) {
// getIndicatorCount return 0 if auxGroupSet is not empty
- val agg = otherAggCalls.get(bit - (fullGroupSet.length + aggRel.getIndicatorCount))
+ val agg = otherAggCalls.asJava.get(bit - (fullGroupSet.length + aggRel.getIndicatorCount))
aggCalls += agg
})
(childKeyBuilder.build(), aggCalls.toArray)
@@ -382,15 +385,16 @@ object FlinkRelMdUtil {
val containsAllAggGroupKeys = fullGroupSet.indices.forall(groupKey.get)
val childKeyBuilder = ImmutableBitSet.builder
val aggs = new mutable.ArrayBuffer[AggregateCall]()
- groupKey.foreach {
- bit =>
- if (bit < fullGroupSet.length) {
- childKeyBuilder.set(fullGroupSet(bit))
- } else if (!containsAllAggGroupKeys) {
- val agg = aggCalls.get(bit - fullGroupSet.length)
- aggs += agg
- }
- }
+ groupKey
+ .stream()
+ .forEach(
+ bit =>
+ if (bit < fullGroupSet.length) {
+ childKeyBuilder.set(fullGroupSet(bit))
+ } else if (!containsAllAggGroupKeys) {
+ val agg = aggCalls.asJava.get(bit - fullGroupSet.length)
+ aggs += agg
+ })
(childKeyBuilder.build(), aggs.toArray)
}
@@ -571,7 +575,7 @@ object FlinkRelMdUtil {
val aggInputFields = agg.getInput.getRowType.getFieldList
val adjustments = new Array[Int](aggOutputFields.size)
grouping.zipWithIndex.foreach { case (bit, index) => adjustments(index) = bit - index }
- val pushableConditions = pushable.map {
+ val pushableConditions = pushable.asScala.map {
pushCondition =>
pushCondition.accept(
new RelOptUtil.RexInputConverter(
@@ -580,7 +584,7 @@ object FlinkRelMdUtil {
aggInputFields,
adjustments))
}
- Option(RexUtil.composeConjunction(rexBuilder, pushableConditions, true))
+ Option(RexUtil.composeConjunction(rexBuilder, pushableConditions.asJava, true))
}
val restPred = if (notPushable.isEmpty) {
None
@@ -596,7 +600,7 @@ object FlinkRelMdUtil {
val mq = rel.getCluster.getMetadataQuery
val columnSizes = mq.getAverageColumnSizes(rel)
var length = 0d
- columnSizes.zip(binaryType.getChildren).foreach {
+ columnSizes.asScala.zip(binaryType.getChildren.asScala).foreach {
case (columnSize, internalType) =>
if (BinaryRowData.isInFixedLengthPart(internalType)) {
length += 8
@@ -664,7 +668,7 @@ object FlinkRelMdUtil {
*/
def getInputRefIndices(index: Int, expand: Expand): util.Set[Int] = {
val inputRefs = new util.HashSet[Int]()
- for (project <- expand.projects) {
+ for (project <- expand.projects.asScala) {
project.get(index) match {
case inputRef: RexInputRef => inputRefs.add(inputRef.getIndex)
case _ => inputRefs.add(-1)
@@ -679,7 +683,7 @@ object FlinkRelMdUtil {
columns: ImmutableBitSet): (ImmutableBitSet, ImmutableBitSet) = {
val leftBuilder = ImmutableBitSet.builder
val rightBuilder = ImmutableBitSet.builder
- columns.foreach {
+ columns.asScala.foreach {
bit => if (bit < leftCount) leftBuilder.set(bit) else rightBuilder.set(bit - leftCount)
}
(leftBuilder.build, rightBuilder.build)
(flink) 08/08: [FLINK-39859][table] Upgrade Calcite to 1.41.0
Posted by sn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
snuyanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit c73b956225ae1791804ff9a0b7f00580a8c49f2a
Author: Sergey Nuyanzin <sn...@gmail.com>
AuthorDate: Wed Jun 3 08:01:44 2026 +0200
[FLINK-39859][table] Upgrade Calcite to 1.41.0
This closes #28316.
---
flink-table/flink-sql-parser/pom.xml | 13 +-
.../src/main/codegen/data/Parser.tdd | 1 +
.../src/main/codegen/templates/Parser.jj | 60 +-
.../org/apache/calcite/sql/SqlMapTypeNameSpec.java | 9 +-
.../sql/parser/validate/FlinkSqlConformance.java | 10 +
flink-table/flink-table-calcite-bridge/pom.xml | 10 +-
flink-table/flink-table-planner/pom.xml | 13 +
.../calcite/rel/rules/SubQueryRemoveRule.java | 150 +-
.../calcite/rel/type/RelDataTypeFactoryImpl.java | 16 +-
.../java/org/apache/calcite/rex/RexSimplify.java | 3715 ++++++++++++++++++++
.../main/java/org/apache/calcite/rex/RexUtil.java | 134 +-
.../org/apache/calcite/runtime/SqlFunctions.java | 502 ++-
.../calcite/sql/fun/SqlCoalesceFunction.java | 9 +-
.../calcite/sql/fun/SqlStdOperatorTable.java | 79 +-
.../calcite/sql/type/SqlTypeFactoryImpl.java | 6 +-
.../org/apache/calcite/sql/type/SqlTypeUtil.java | 219 +-
.../calcite/sql/validate/SqlValidatorImpl.java | 395 ++-
.../apache/calcite/sql2rel/RelDecorrelator.java | 333 +-
.../apache/calcite/sql2rel/SqlToRelConverter.java | 266 +-
.../plan/nodes/exec/common/CommonExecMatch.java | 7 +
...rrelateToJoinFromTemporalTableFunctionRule.java | 6 +
.../rules/logical/RemoteCorrelateSplitRule.java | 6 +
.../SplitPythonConditionFromCorrelateRule.java | 6 +
.../physical/common/CommonPhysicalMatchRule.java | 6 +
.../flink/table/planner/plan/utils/AsyncUtil.java | 6 +
.../planner/plan/utils/ConstantFoldingUtil.java | 6 +
.../flink/table/planner/plan/utils/PythonUtil.java | 11 +
.../src/main/resources/META-INF/NOTICE | 8 +-
.../table/planner/codegen/ExprCodeGenerator.scala | 5 +
.../plan/rules/logical/RemoteCalcSplitRule.scala | 11 +-
.../flink/table/planner/plan/utils/MatchUtil.scala | 7 +-
.../planner/plan/utils/RexNodeExtractor.scala | 5 +
.../table/planner/plan/batch/sql/ValuesTest.xml | 6 +-
.../plan/batch/sql/agg/GroupingSetsTest.xml | 22 +-
.../plan/batch/sql/join/BroadcastHashJoinTest.xml | 2 +-
.../plan/batch/sql/join/NestedLoopJoinTest.xml | 2 +-
.../plan/batch/sql/join/ShuffledHashJoinTest.xml | 2 +-
.../plan/batch/sql/join/SortMergeJoinTest.xml | 2 +-
.../logical/DecomposeGroupingSetsRuleTest.xml | 13 +-
.../plan/rules/logical/FlinkFilterJoinRuleTest.xml | 4 +-
.../logical/RemoveSingleAggregateRuleTest.xml | 18 +-
.../logical/SimplifyFilterConditionRuleTest.xml | 4 +-
.../logical/SimplifyJoinConditionRuleTest.xml | 20 +-
.../logical/subquery/SubQueryAntiJoinTest.xml | 99 +-
.../logical/subquery/SubQuerySemiJoinTest.xml | 42 +-
.../plan/stream/sql/InsertIntoValuesTest.xml | 4 +-
.../table/planner/plan/stream/sql/ValuesTest.xml | 6 +-
.../plan/stream/sql/agg/GroupingSetsTest.xml | 36 +
.../metadata/FlinkRelMdDistinctRowCountTest.scala | 19 +-
.../metadata/FlinkRelMdPopulationSizeTest.scala | 4 +-
.../plan/stream/sql/agg/GroupingSetsTest.scala | 6 +-
.../runtime/batch/sql/agg/GroupingSetsITCase.scala | 7 +-
flink-table/pom.xml | 2 +-
53 files changed, 5855 insertions(+), 495 deletions(-)
diff --git a/flink-table/flink-sql-parser/pom.xml b/flink-table/flink-sql-parser/pom.xml
index 2c4c37d47bb..9a682e56e08 100644
--- a/flink-table/flink-sql-parser/pom.xml
+++ b/flink-table/flink-sql-parser/pom.xml
@@ -68,14 +68,15 @@ under the License.
<version>${calcite.version}</version>
<exclusions>
<!--
- "mvn dependency:tree" as of Calcite 1.40.0:
+ "mvn dependency:tree" as of Calcite 1.41.0:
- [INFO] +- org.apache.calcite:calcite-core:jar:1.40.0:compile
- [INFO] | +- org.apache.calcite:calcite-linq4j:jar:1.40.0:compile
- [INFO] | +- org.apache.calcite.avatica:avatica-core:jar:1.26.0:compile
+ [INFO] +- org.apache.calcite:calcite-core:jar:1.41.0:compile
+ [INFO] | +- org.apache.calcite:calcite-linq4j:jar:1.41.0:compile
+ [INFO] | +- org.apache.calcite.avatica:avatica-core:jar:1.27.0:compile
[INFO] | +- org.apiguardian:apiguardian-api:jar:1.1.2:compile
- [INFO] | +- org.checkerframework:checker-qual:jar:3.43.0:compile
- [INFO] | \- org.apache.commons:commons-math3:jar:3.6.1:runtime
+ [INFO] | +- org.checkerframework:checker-qual:jar:3.10.0:compile
+ [INFO] | +- org.apache.commons:commons-math3:jar:3.6.1:runtime
+ [INFO] | \- org.jooq:joou-java-6:jar:0.9.4:compile
Dependencies that are not needed for how we use Calcite right now.
-->
diff --git a/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd b/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd
index 84b69813a52..b7e6d7ab9ea 100644
--- a/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd
+++ b/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd
@@ -781,4 +781,5 @@
includeBraces: true
includeAdditionalDeclarations: false
includeParsingStringLiteralAsArrayLiteral: false
+ includeIntervalWithoutQualifier: false
}
diff --git a/flink-table/flink-sql-parser/src/main/codegen/templates/Parser.jj b/flink-table/flink-sql-parser/src/main/codegen/templates/Parser.jj
index 98b5004a374..cb9121e97d1 100644
--- a/flink-table/flink-sql-parser/src/main/codegen/templates/Parser.jj
+++ b/flink-table/flink-sql-parser/src/main/codegen/templates/Parser.jj
@@ -5246,6 +5246,19 @@ TimeUnit Second() :
<SECONDS> { return warn(TimeUnit.SECOND); }
}
+SqlIntervalQualifier IntervalWithoutQualifier() :
+{
+ final Span s;
+ int secondFracPrec = RelDataType.PRECISION_NOT_SPECIFIED;
+}
+{
+ {
+ s = span();
+ return new SqlIntervalQualifier(TimeUnit.SECOND, -1, null, secondFracPrec,
+ s.end(this));
+ }
+}
+
SqlIntervalQualifier IntervalQualifier() :
{
final Span s;
@@ -5905,6 +5918,7 @@ SqlTypeNameSpec SqlTypeName(Span s) :
SqlTypeNameSpec SqlTypeName1(Span s) :
{
final SqlTypeName sqlTypeName;
+ boolean unsigned = false;
}
{
(
@@ -5918,13 +5932,40 @@ SqlTypeNameSpec SqlTypeName1(Span s) :
|
<BOOLEAN> { s.add(this); sqlTypeName = SqlTypeName.BOOLEAN; }
|
- ( <INTEGER> | <INT> ) { s.add(this); sqlTypeName = SqlTypeName.INTEGER; }
+ ( <INTEGER> | <INT> ) ( <UNSIGNED> { unsigned = true; })? {
+ if (unsigned && !this.conformance.supportsUnsignedTypes()) {
+ throw SqlUtil.newContextException(getPos(), RESOURCE.unsignedDisabled());
+ }
+ s.add(this); sqlTypeName = unsigned ? SqlTypeName.UINTEGER : SqlTypeName.INTEGER;
+ }
|
- <TINYINT> { s.add(this); sqlTypeName = SqlTypeName.TINYINT; }
+ <UNSIGNED> {
+ if (!this.conformance.supportsUnsignedTypes()) {
+ throw SqlUtil.newContextException(getPos(), RESOURCE.unsignedDisabled());
+ }
+ s.add(this); sqlTypeName = SqlTypeName.UINTEGER;
+ }
+ |
+ <TINYINT> ( <UNSIGNED> { unsigned = true; })? {
+ if (unsigned && !this.conformance.supportsUnsignedTypes()) {
+ throw SqlUtil.newContextException(getPos(), RESOURCE.unsignedDisabled());
+ }
+ s.add(this); sqlTypeName = unsigned ? SqlTypeName.UTINYINT : SqlTypeName.TINYINT;
+ }
|
- <SMALLINT> { s.add(this); sqlTypeName = SqlTypeName.SMALLINT; }
+ <SMALLINT> ( <UNSIGNED> { unsigned = true; })? {
+ if (unsigned && !this.conformance.supportsUnsignedTypes()) {
+ throw SqlUtil.newContextException(getPos(), RESOURCE.unsignedDisabled());
+ }
+ s.add(this); sqlTypeName = unsigned ? SqlTypeName.USMALLINT : SqlTypeName.SMALLINT;
+ }
|
- <BIGINT> { s.add(this); sqlTypeName = SqlTypeName.BIGINT; }
+ <BIGINT> ( <UNSIGNED> { unsigned = true; })? {
+ if (unsigned && !this.conformance.supportsUnsignedTypes()) {
+ throw SqlUtil.newContextException(getPos(), RESOURCE.unsignedDisabled());
+ }
+ s.add(this); sqlTypeName = unsigned ? SqlTypeName.UBIGINT : SqlTypeName.BIGINT;
+ }
|
<REAL> { s.add(this); sqlTypeName = SqlTypeName.REAL; }
|
@@ -6310,7 +6351,12 @@ SqlNode BuiltinFunctionCall() :
(
dt = DataType() { args.add(dt); }
|
+ LOOKAHEAD(2)
<INTERVAL> e = IntervalQualifier() { args.add(e); }
+<#if (parser.includeIntervalWithoutQualifier!default.parser.includeIntervalWithoutQualifier) >
+ |
+ <INTERVAL> e = IntervalWithoutQualifier() { args.add(e); }
+</#if>
)
[ <FORMAT> format = StringLiteral() { args.add(format); } ]
<RPAREN> {
@@ -7957,6 +8003,7 @@ SqlBinaryOperator BinaryRowOperator() :
{
// <IN> is handled as a special case
<EQ> { return SqlStdOperatorTable.EQUALS; }
+| <LEFTSHIFT> { return SqlStdOperatorTable.BIT_LEFT_SHIFT; }
| <GT> { return SqlStdOperatorTable.GREATER_THAN; }
| <LT> { return SqlStdOperatorTable.LESS_THAN; }
| <LE> { return SqlStdOperatorTable.LESS_THAN_OR_EQUAL; }
@@ -7988,7 +8035,9 @@ SqlBinaryOperator BinaryRowOperator() :
| <NOT> <SUBMULTISET> <OF> { return SqlStdOperatorTable.NOT_SUBMULTISET_OF; }
| <CONTAINS> { return SqlStdOperatorTable.CONTAINS; }
| <OVERLAPS> { return SqlStdOperatorTable.OVERLAPS; }
+| <CARET> { return SqlStdOperatorTable.BITXOR_OPERATOR; }
| <EQUALS> { return SqlStdOperatorTable.PERIOD_EQUALS; }
+| <AMPERSAND> { return SqlStdOperatorTable.BITAND_OPERATOR; }
| <PRECEDES> { return SqlStdOperatorTable.PRECEDES; }
| <SUCCEEDS> { return SqlStdOperatorTable.SUCCEEDS; }
| LOOKAHEAD(2) <IMMEDIATELY> <PRECEDES> { return SqlStdOperatorTable.IMMEDIATELY_PRECEDES; }
@@ -8735,6 +8784,7 @@ SqlPostfixOperator PostfixRowOperator() :
| < UNPIVOT: "UNPIVOT" >
| < UNNAMED: "UNNAMED" >
| < UNNEST: "UNNEST" >
+| < UNSIGNED: "UNSIGNED" >
| < UPDATE: "UPDATE" > { beforeTableName(); }
| < UPPER: "UPPER" >
| < UPSERT: "UPSERT" >
@@ -8978,6 +9028,8 @@ void NonReservedKeyWord2of3() :
| < DOUBLE_QUOTE: "\"" >
| < VERTICAL_BAR: "|" >
| < CARET: "^" >
+| < AMPERSAND: "&" >
+| < LEFTSHIFT: "<<" >
| < DOLLAR: "$" >
<#list (parser.binaryOperatorsTokens!default.parser.binaryOperatorsTokens) as operator>
| ${operator}
diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/calcite/sql/SqlMapTypeNameSpec.java b/flink-table/flink-sql-parser/src/main/java/org/apache/calcite/sql/SqlMapTypeNameSpec.java
index 827302de6a2..616dff23490 100644
--- a/flink-table/flink-sql-parser/src/main/java/org/apache/calcite/sql/SqlMapTypeNameSpec.java
+++ b/flink-table/flink-sql-parser/src/main/java/org/apache/calcite/sql/SqlMapTypeNameSpec.java
@@ -57,9 +57,12 @@ public class SqlMapTypeNameSpec extends SqlTypeNameSpec {
@Override
public RelDataType deriveType(SqlValidator validator) {
- return validator
- .getTypeFactory()
- .createMapType(keyType.deriveType(validator), valType.deriveType(validator));
+ boolean keyCanBeNullable =
+ validator.getTypeFactory().getTypeSystem().mapKeysCanBeNullable();
+ RelDataType kType = keyType.deriveType(validator, keyCanBeNullable);
+
+ RelDataType valueType = valType.deriveType(validator, true);
+ return validator.getTypeFactory().createMapType(kType, valueType);
}
@Override
diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/validate/FlinkSqlConformance.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/validate/FlinkSqlConformance.java
index 0524581f2a3..ccc1ef14f60 100644
--- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/validate/FlinkSqlConformance.java
+++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/validate/FlinkSqlConformance.java
@@ -47,6 +47,11 @@ public enum FlinkSqlConformance implements SqlConformance {
return SqlConformanceEnum.DEFAULT.isSelectAlias();
}
+ @Override
+ public boolean isNonStrictGroupBy() {
+ return false;
+ }
+
@Override
public boolean isGroupByOrdinal() {
return false;
@@ -192,6 +197,11 @@ public enum FlinkSqlConformance implements SqlConformance {
return SqlConformanceEnum.DEFAULT.checkedArithmetic();
}
+ @Override
+ public boolean supportsUnsignedTypes() {
+ return false;
+ }
+
@Override
public boolean isValueAllowed() {
return true;
diff --git a/flink-table/flink-table-calcite-bridge/pom.xml b/flink-table/flink-table-calcite-bridge/pom.xml
index 90ade1deb93..77ee0a2fd49 100644
--- a/flink-table/flink-table-calcite-bridge/pom.xml
+++ b/flink-table/flink-table-calcite-bridge/pom.xml
@@ -45,18 +45,18 @@ under the License.
<version>${calcite.version}</version>
<exclusions>
<!--
- "mvn dependency:tree" as of Calcite 1.40.0:
- [INFO] org.apache.flink:flink-table-calcite-bridge:jar:2.4-SNAPSHOT
- [INFO] +- org.apache.calcite:calcite-core:jar:1.40.0:compile
- [INFO] | +- org.apache.calcite:calcite-linq4j:jar:1.40.0:compile
+ "mvn dependency:tree" as of Calcite 1.41.0:
+ [INFO] +- org.apache.calcite:calcite-core:jar:1.41.0:compile
+ [INFO] | +- org.apache.calcite:calcite-linq4j:jar:1.41.0:compile
[INFO] | +- org.locationtech.jts:jts-core:jar:1.19.0:compile
[INFO] | +- com.fasterxml.jackson.core:jackson-annotations:jar:2.21:compile
- [INFO] | +- org.apache.calcite.avatica:avatica-core:jar:1.26.0:compile
+ [INFO] | +- org.apache.calcite.avatica:avatica-core:jar:1.27.0:compile
[INFO] | +- org.apiguardian:apiguardian-api:jar:1.1.2:compile
[INFO] | +- com.fasterxml.jackson.core:jackson-core:jar:2.21.3:compile
[INFO] | +- com.fasterxml.jackson.core:jackson-databind:jar:2.21.3:compile
[INFO] | +- commons-codec:commons-codec:jar:1.15:runtime
[INFO] | +- org.apache.commons:commons-math3:jar:3.6.1:runtime
+ [INFO] | +- org.jooq:joou-java-6:jar:0.9.4:compile
[INFO] | \- commons-io:commons-io:jar:2.15.1:runtime
[INFO] +- com.jayway.jsonpath:json-path:jar:2.9.0:compile
[INFO] | \- net.minidev:json-smart:jar:2.5.0:runtime
diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml
index e37ffce9569..b8f76f7f969 100644
--- a/flink-table/flink-table-planner/pom.xml
+++ b/flink-table/flink-table-planner/pom.xml
@@ -144,6 +144,12 @@ under the License.
<version>3.12.0</version>
<optional>${flink.markBundledAsOptional}</optional>
</dependency>
+ <dependency>
+ <groupId>org.jooq</groupId>
+ <artifactId>joou-java-6</artifactId>
+ <version>0.9.4</version>
+ <optional>${flink.markBundledAsOptional}</optional>
+ </dependency>
<!-- Table Runtime (not included in the uber) -->
@@ -371,6 +377,7 @@ under the License.
<include>commons-io:commons-io</include>
<include>org.apache.commons:commons-math3</include>
<include>org.checkerframework:checker-qual</include>
+ <include>org.jooq:joou-java-6</include>
<!-- flink-table-planner dependencies -->
<include>org.apache.flink:flink-sql-parser</include>
@@ -425,6 +432,12 @@ under the License.
<pattern>com.ibm.icu</pattern>
<shadedPattern>org.apache.flink.table.shaded.com.ibm.icu</shadedPattern>
</relocation>
+
+ <relocation>
+ <!-- org.jooq's dependencies, required for unsigned types -->
+ <pattern>org.jooq</pattern>
+ <shadedPattern>org.apache.flink.table.shaded.org.jooq</shadedPattern>
+ </relocation>
</relocations>
</configuration>
</plugin>
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java
index 0114ad194fd..09fc596c5ce 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rel/rules/SubQueryRemoveRule.java
@@ -17,9 +17,11 @@
package org.apache.calcite.rel.rules;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.RelRule;
+import org.apache.calcite.rel.RelHomogeneousShuttle;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Collect;
import org.apache.calcite.rel.core.CorrelationId;
@@ -30,6 +32,7 @@ import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.metadata.RelMdUtil;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rex.LogicVisitor;
+import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
@@ -57,7 +60,7 @@ import static org.apache.calcite.util.Util.last;
* Default implementation of {@link org.apache.calcite.rel.rules.SubQueryRemoveRule}, the class was
* copied over because of LITERAL_AGG function which should be implemented in Flink first.
*
- * <p>Lines 740 ~ 744, Use Calcite 1.34.0 behavior.
+ * <p>Lines 781 ~ 785, Use Calcite 1.34.0 behavior.
*/
@Value.Enclosing
public class SubQueryRemoveRule extends RelRule<SubQueryRemoveRule.Config>
@@ -742,10 +745,29 @@ public class SubQueryRemoveRule extends RelRule<SubQueryRemoveRule.Config>
case TRUE_FALSE_UNKNOWN:
case UNKNOWN_AS_TRUE:
// Builds the cross join
- builder.aggregate(
- builder.groupKey(),
- builder.count(false, "c"),
- builder.count(builder.fields()).as("ck"));
+ // Some databases don't support use FILTER clauses for aggregate functions
+ // like {@code COUNT(*) FILTER (WHERE not(a is null))}
+ // So use count(*) when only one column
+ if (builder.fields().size() <= 1) {
+ builder.aggregate(
+ builder.groupKey(),
+ builder.count(false, "c"),
+ builder.count(builder.fields()).as("ck"));
+ } else {
+ builder.aggregate(
+ builder.groupKey(),
+ builder.count(false, "c"),
+ builder.count()
+ .filter(
+ builder.not(
+ builder.and(
+ builder.fields().stream()
+ .map(builder::isNull)
+ .collect(
+ Collectors
+ .toList()))))
+ .as("ck"));
+ }
builder.as(ctAlias);
if (!variablesSet.isEmpty()) {
builder.join(JoinRelType.LEFT, trueLiteral, variablesSet);
@@ -925,13 +947,30 @@ public class SubQueryRemoveRule extends RelRule<SubQueryRemoveRule.Config>
final Join join = call.rel(0);
final RelBuilder builder = call.builder();
final RexSubQuery e = requireNonNull(RexUtil.SubQueryFinder.find(join.getCondition()));
- final RelOptUtil.Logic logic =
- LogicVisitor.find(RelOptUtil.Logic.TRUE, ImmutableList.of(join.getCondition()), e);
ImmutableBitSet inputSet = RelOptUtil.InputFinder.bits(e.getOperands(), null);
int nFieldsLeft = join.getLeft().getRowType().getFieldCount();
int nFieldsRight = join.getRight().getRowType().getFieldCount();
+ // Correlation columns should also be considered.
+ // For example:
+ // LogicalJoin
+ // left right
+ // | |
+ // LogicalProject.NONE.[0, 1] LogicalValues.NONE.[0]
+ // RecordType(INTEGER DEPTNO, CHAR(11) DNAME) RecordType(INTEGER DEPTNO)
+ //
+ // and subquery: $SCALAR_QUERY with correlate
+ // LogicalProject(DEPTNO=[$1])
+ // LogicalFilter(condition=[=(CAST($0):CHAR(11) NOT NULL, $cor0.DNAME)])
+ //
+ // In such a case $cor0.DNAME need to be accounted as input form left side.
+ final Set<CorrelationId> variablesSet = RelOptUtil.getVariablesUsed(e.rel);
+ for (CorrelationId id : variablesSet) {
+ ImmutableBitSet requiredColumns = RelOptUtil.correlationColumns(id, e.rel);
+ inputSet = ImmutableBitSet.union(ImmutableList.of(requiredColumns, inputSet));
+ }
+
boolean inputIntersectsLeftSide =
inputSet.intersects(ImmutableBitSet.range(0, nFieldsLeft));
boolean inputIntersectsRightSide =
@@ -945,10 +984,17 @@ public class SubQueryRemoveRule extends RelRule<SubQueryRemoveRule.Config>
return;
}
- final Set<CorrelationId> variablesSet = RelOptUtil.getVariablesUsed(e.rel);
if (inputIntersectsLeftSide) {
builder.push(join.getLeft());
+ final RelOptUtil.Logic logic =
+ LogicVisitor.find(
+ join.getJoinType().generatesNullsOnRight()
+ ? RelOptUtil.Logic.TRUE_FALSE_UNKNOWN
+ : RelOptUtil.Logic.TRUE,
+ ImmutableList.of(join.getCondition()),
+ e);
+
final RexNode target = rule.apply(e, variablesSet, logic, builder, 1, nFieldsLeft, 0);
final RexShuttle shuttle = new ReplaceSubQueryShuttle(e, target);
@@ -968,12 +1014,94 @@ public class SubQueryRemoveRule extends RelRule<SubQueryRemoveRule.Config>
.union(ImmutableBitSet.range(nFields - nFieldsRight, nFields)));
builder.project(fields);
} else {
- builder.push(join.getLeft());
builder.push(join.getRight());
+ final RelOptUtil.Logic logic =
+ LogicVisitor.find(
+ join.getJoinType().generatesNullsOnLeft()
+ ? RelOptUtil.Logic.TRUE_FALSE_UNKNOWN
+ : RelOptUtil.Logic.TRUE,
+ ImmutableList.of(join.getCondition()),
+ e);
+
+ RexSubQuery subQuery = e;
+
+ if (!variablesSet.isEmpty()) {
+ // Original correlates reference joint row type, but we are about to create
+ // new join of original right side and correlated sub-query. Therefore we have
+ // to adjust correlated variables in following way:
+ // 1) new correlation variable must reference row type of right side only
+ // 2) field index must be shifted on the size of the left side
+ // Example:
+ // SELECT e1.*
+ // FROM emp e1
+ // JOIN dept d
+ // ON e1.deptno = d.deptno
+ // AND d.deptno IN (
+ // SELECT e3.empno
+ // FROM emp e3
+ // WHERE d.deptno > e3.comm
+ // )
+ // ORDER BY e1.empno, e1.deptno;
+ //
+ // LogicalJoin(condition=[AND(=($7, $8), IN(CAST($8):SMALLINT NOT NULL, {
+ // LogicalProject(EMPNO=[$0])
+ // LogicalFilter(condition=[>(CAST($cor0.DEPTNO0):DECIMAL(7, 2) NOT NULL, $6)])
+ // LogicalTableScan(table=[[scott, EMP]])
+ // }))], joinType=[inner])
+ // LogicalTableScan(table=[[scott, EMP]])
+ // LogicalProject(DEPTNO=[$0])
+ // LogicalTableScan(table=[[scott, DEPT]])
+ //
+ // Rewrite to:
+ //
+ // LogicalProject(EMPNO=[$0], ENAME=[$1], ..., COMM=[$6], DEPTNO=[$7], DEPTNO0=[$8])
+ // LogicalJoin(condition=[=($7, $8)], joinType=[inner])
+ // LogicalTableScan(table=[[scott, EMP]])
+ // LogicalFilter(condition=[=(CAST($0):SMALLINT NOT NULL, $1)])
+ // LogicalCorrelate(correlation=[$cor0], joinType=[inner],
+ // requiredColumns=[{0}])
+ // LogicalProject(DEPTNO=[$0])
+ // LogicalTableScan(table=[[scott, DEPT]])
+ // LogicalProject(EMPNO=[$0])
+ // LogicalFilter(condition=[>(CAST($cor0.DEPTNO):DECIMAL(7, 2) NOT NULL,
+ // $6)])
+ // LogicalTableScan(table=[[scott, EMP]])
+ CorrelationId id = Iterables.getOnlyElement(variablesSet);
+ RexBuilder rexBuilder = builder.getRexBuilder();
+
+ RelNode newSubQueryRel =
+ e.rel.accept(
+ new RelHomogeneousShuttle() {
+ @Override
+ public RelNode visit(RelNode other) {
+ RelNode node =
+ RexUtil.shiftFieldAccess(
+ rexBuilder,
+ other,
+ id,
+ join.getRight(),
+ -nFieldsLeft);
+ return super.visit(node);
+ }
+ });
+ subQuery = e.clone(newSubQueryRel);
+ }
+
+ subQuery =
+ subQuery.clone(
+ subQuery.getType(),
+ RexUtil.shift(subQuery.getOperands(), -nFieldsLeft));
+
final int nFields = join.getRowType().getFieldCount();
- final RexNode target = rule.apply(e, variablesSet, logic, builder, 2, nFields, 0);
- final RexShuttle shuttle = new ReplaceSubQueryShuttle(e, target);
+ final RexNode target =
+ rule.apply(subQuery, variablesSet, logic, builder, 1, nFieldsRight, 0);
+ final RexShuttle shuttle =
+ new ReplaceSubQueryShuttle(e, RexUtil.shift(target, nFieldsLeft));
+
+ RelNode newRight = builder.build();
+ builder.push(join.getLeft());
+ builder.push(newRight);
builder.join(join.getJoinType(), shuttle.apply(join.getCondition()));
builder.project(fields(builder, nFields));
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java
index 0fbbb0be86d..6ab2d2e2a15 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java
@@ -58,8 +58,8 @@ import static org.apache.calcite.util.ReflectUtil.isStatic;
* <p>FLINK modifications are at lines
*
* <ol>
- * <li>Should be removed after fixing CALCITE-5199: Lines 241-243
- * <li>Preserve FLOAT precision as in Calcite 1.38.0: Lines 592 ~ 594
+ * <li>Added in FLINK-39695 (backport of CALCITE-6764): Lines 406 ~ 432
+ * <li>Keep Flink FLOAT precision: Lines 592 ~ 594
* </ol>
*/
public abstract class RelDataTypeFactoryImpl implements RelDataTypeFactory {
@@ -238,9 +238,7 @@ public abstract class RelDataTypeFactoryImpl implements RelDataTypeFactory {
// recursively compute column-wise least restrictive
// preserve the struct kind from type0
- // FLINK MODIFICATION BEGIN
final Builder builder = builder().kind(type0.getStructKind());
- // FLINK MODIFICATION END
for (int j = 0; j < fieldCount; ++j) {
// REVIEW jvs 22-Jan-2004: Always use the field name from the
// first type?
@@ -277,9 +275,11 @@ public abstract class RelDataTypeFactoryImpl implements RelDataTypeFactory {
if (type == null) {
return null;
}
- return sqlTypeName == SqlTypeName.ARRAY
- ? new ArraySqlType(type, isNullable)
- : new MultisetSqlType(type, isNullable);
+ RelDataType collection =
+ sqlTypeName == SqlTypeName.ARRAY
+ ? createArrayType(type, -1)
+ : createMultisetType(type, -1);
+ return createTypeWithNullability(collection, isNullable);
}
protected @Nullable RelDataType leastRestrictiveMapType(
@@ -302,7 +302,7 @@ public abstract class RelDataTypeFactoryImpl implements RelDataTypeFactory {
if (valueType == null) {
return null;
}
- return new MapSqlType(keyType, valueType, isNullable);
+ return createTypeWithNullability(createMapType(keyType, valueType), isNullable);
}
protected RelDataType leastRestrictiveIntervalDatetimeType(
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rex/RexSimplify.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rex/RexSimplify.java
new file mode 100644
index 00000000000..911a325f4ac
--- /dev/null
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rex/RexSimplify.java
@@ -0,0 +1,3715 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.rex;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.BoundType;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableRangeSet;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Range;
+import com.google.common.collect.RangeSet;
+import com.google.common.collect.Sets;
+import com.google.common.collect.TreeRangeSet;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.avatica.util.TimeUnitRange;
+import org.apache.calcite.plan.RelOptPredicateList;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.Strong;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.metadata.NullSentinel;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeCoercionRule;
+import org.apache.calcite.sql.type.SqlTypeFamily;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.type.SqlTypeUtil;
+import org.apache.calcite.util.Bug;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.RangeSets;
+import org.apache.calcite.util.Sarg;
+import org.apache.calcite.util.Util;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.calcite.linq4j.Nullness.castNonNull;
+import static org.apache.calcite.rex.RexUnknownAs.FALSE;
+import static org.apache.calcite.rex.RexUnknownAs.TRUE;
+import static org.apache.calcite.rex.RexUnknownAs.UNKNOWN;
+
+/**
+ * Context required to simplify a row-expression.
+ *
+ * <p>FLINK modifications are at lines
+ *
+ * <ol>
+ * <li>CALCITE-7588, CALCITE-7578: Lines 574 ~ 588, 598 ~ 600, 624 ~ 636
+ * </ol>
+ */
+public class RexSimplify {
+ private final boolean paranoid;
+ public final RexBuilder rexBuilder;
+ private final RelOptPredicateList predicates;
+
+ /**
+ * How to treat UNKNOWN values, if one of the deprecated {@code simplify} methods without an
+ * {@code unknownAs} argument is called.
+ */
+ final RexUnknownAs defaultUnknownAs;
+
+ final boolean predicateElimination;
+ private final RexExecutor executor;
+
+ private static final Strong STRONG = new Strong();
+
+ /**
+ * Creates a RexSimplify.
+ *
+ * @param rexBuilder Rex builder
+ * @param predicates Predicates known to hold on input fields
+ * @param executor Executor for constant reduction, not null
+ */
+ public RexSimplify(
+ RexBuilder rexBuilder, RelOptPredicateList predicates, RexExecutor executor) {
+ this(rexBuilder, predicates, UNKNOWN, true, false, executor);
+ }
+
+ /** Internal constructor. */
+ private RexSimplify(
+ RexBuilder rexBuilder,
+ RelOptPredicateList predicates,
+ RexUnknownAs defaultUnknownAs,
+ boolean predicateElimination,
+ boolean paranoid,
+ RexExecutor executor) {
+ this.rexBuilder = requireNonNull(rexBuilder, "rexBuilder");
+ this.predicates = requireNonNull(predicates, "predicates");
+ this.defaultUnknownAs = requireNonNull(defaultUnknownAs, "defaultUnknownAs");
+ this.predicateElimination = predicateElimination;
+ this.paranoid = paranoid;
+ this.executor = requireNonNull(executor, "executor");
+ }
+
+ @Deprecated // to be removed before 2.0
+ public RexSimplify(RexBuilder rexBuilder, boolean unknownAsFalse, RexExecutor executor) {
+ this(
+ rexBuilder,
+ RelOptPredicateList.EMPTY,
+ RexUnknownAs.falseIf(unknownAsFalse),
+ true,
+ false,
+ executor);
+ }
+
+ @Deprecated // to be removed before 2.0
+ public RexSimplify(
+ RexBuilder rexBuilder,
+ RelOptPredicateList predicates,
+ boolean unknownAsFalse,
+ RexExecutor executor) {
+ this(rexBuilder, predicates, RexUnknownAs.falseIf(unknownAsFalse), true, false, executor);
+ }
+
+ // ~ Methods ----------------------------------------------------------------
+
+ /**
+ * Returns a RexSimplify the same as this but with a specified {@link #defaultUnknownAs} value.
+ *
+ * @deprecated Use methods with a {@link RexUnknownAs} argument, such as {@link
+ * #simplify(RexNode, RexUnknownAs)}.
+ */
+ @Deprecated // to be removed before 2.0
+ public RexSimplify withUnknownAsFalse(boolean unknownAsFalse) {
+ final RexUnknownAs defaultUnknownAs = RexUnknownAs.falseIf(unknownAsFalse);
+ return defaultUnknownAs == this.defaultUnknownAs
+ ? this
+ : new RexSimplify(
+ rexBuilder,
+ predicates,
+ defaultUnknownAs,
+ predicateElimination,
+ paranoid,
+ executor);
+ }
+
+ /** Returns a RexSimplify the same as this but with a specified {@link #predicates} value. */
+ public RexSimplify withPredicates(RelOptPredicateList predicates) {
+ return predicates == this.predicates
+ ? this
+ : new RexSimplify(
+ rexBuilder,
+ predicates,
+ defaultUnknownAs,
+ predicateElimination,
+ paranoid,
+ executor);
+ }
+
+ /**
+ * Returns a RexSimplify the same as this but which verifies that the expression before and
+ * after simplification are equivalent.
+ *
+ * @see #verify
+ */
+ public RexSimplify withParanoid(boolean paranoid) {
+ return paranoid == this.paranoid
+ ? this
+ : new RexSimplify(
+ rexBuilder,
+ predicates,
+ defaultUnknownAs,
+ predicateElimination,
+ paranoid,
+ executor);
+ }
+
+ /**
+ * Returns a RexSimplify the same as this but with a specified {@link #predicateElimination}
+ * value.
+ *
+ * <p>This is introduced temporarily, until {@link Bug#CALCITE_2401_FIXED [CALCITE-2401] is
+ * fixed}.
+ */
+ private RexSimplify withPredicateElimination(boolean predicateElimination) {
+ return predicateElimination == this.predicateElimination
+ ? this
+ : new RexSimplify(
+ rexBuilder,
+ predicates,
+ defaultUnknownAs,
+ predicateElimination,
+ paranoid,
+ executor);
+ }
+
+ /**
+ * Simplifies a boolean expression, always preserving its type and its nullability.
+ *
+ * <p>This is useful if you are simplifying expressions in a {@link Project}.
+ */
+ public RexNode simplifyPreservingType(RexNode e) {
+ return simplifyPreservingType(e, defaultUnknownAs, true);
+ }
+
+ public RexNode simplifyPreservingType(
+ RexNode e, RexUnknownAs unknownAs, boolean matchNullability) {
+ final RexNode e2 = simplifyUnknownAs(e, unknownAs);
+ if (e2.getType() == e.getType()) {
+ return e2;
+ }
+ if (!matchNullability
+ && SqlTypeUtil.equalSansNullability(
+ rexBuilder.typeFactory, e2.getType(), e.getType())) {
+ return e2;
+ }
+ final RexNode e3 = rexBuilder.makeCast(e.getType(), e2, matchNullability, false);
+ if (e3.equals(e)) {
+ return e;
+ }
+ return e3;
+ }
+
+ /**
+ * Simplifies a boolean expression.
+ *
+ * <p>In particular:
+ *
+ * <ul>
+ * <li>{@code simplify(x = 1 OR NOT x = 1 OR x IS NULL)} returns {@code TRUE}
+ * <li>{@code simplify(x = 1 AND FALSE)} returns {@code FALSE}
+ * </ul>
+ *
+ * <p>Handles UNKNOWN values using the policy specified when you created this {@code
+ * RexSimplify}. Unless you used a deprecated constructor, that policy is {@link
+ * RexUnknownAs#UNKNOWN}.
+ *
+ * <p>If the expression is a predicate in a WHERE clause, consider instead using {@link
+ * #simplifyUnknownAsFalse(RexNode)}.
+ *
+ * @param e Expression to simplify
+ */
+ public RexNode simplify(RexNode e) {
+ return simplifyUnknownAs(e, defaultUnknownAs);
+ }
+
+ /**
+ * As {@link #simplify(RexNode)}, but for a boolean expression for which a result of UNKNOWN
+ * will be treated as FALSE.
+ *
+ * <p>Use this form for expressions on a WHERE, ON, HAVING or FILTER(WHERE) clause.
+ *
+ * <p>This may allow certain additional simplifications. A result of UNKNOWN may yield FALSE,
+ * however it may still yield UNKNOWN. (If the simplified expression has type BOOLEAN NOT NULL,
+ * then of course it can only return FALSE.)
+ */
+ public final RexNode simplifyUnknownAsFalse(RexNode e) {
+ return simplifyUnknownAs(e, FALSE);
+ }
+
+ /**
+ * As {@link #simplify(RexNode)}, but specifying how UNKNOWN values are to be treated.
+ *
+ * <p>If UNKNOWN is treated as FALSE, this may allow certain additional simplifications. A
+ * result of UNKNOWN may yield FALSE, however it may still yield UNKNOWN. (If the simplified
+ * expression has type BOOLEAN NOT NULL, then of course it can only return FALSE.)
+ */
+ public RexNode simplifyUnknownAs(RexNode e, RexUnknownAs unknownAs) {
+ final RexNode simplified = withParanoid(false).simplify(e, unknownAs);
+ if (paranoid) {
+ verify(e, simplified, unknownAs);
+ }
+ return simplified;
+ }
+
+ /**
+ * Internal method to simplify an expression.
+ *
+ * <p>Unlike the public {@link #simplify(RexNode)} and {@link #simplifyUnknownAsFalse(RexNode)}
+ * methods, never calls {@link #verify(RexNode, RexNode, RexUnknownAs)}. Verify adds an overhead
+ * that is only acceptable for a top-level call.
+ */
+ RexNode simplify(RexNode e, RexUnknownAs unknownAs) {
+ if (isSafeExpression(e) && STRONG.isNull(e)) {
+ // Only boolean NULL (aka UNKNOWN) can be converted to FALSE. Even in
+ // unknownAs=FALSE mode, we must not convert a NULL integer (say) to FALSE
+ if (e.getType().getSqlTypeName() == SqlTypeName.BOOLEAN) {
+ switch (unknownAs) {
+ case FALSE:
+ case TRUE:
+ return rexBuilder.makeLiteral(unknownAs.toBoolean());
+ default:
+ break;
+ }
+ }
+ return rexBuilder.makeNullLiteral(e.getType());
+ }
+ switch (e.getKind()) {
+ case AND:
+ return simplifyAnd((RexCall) e, unknownAs);
+ case OR:
+ return simplifyOr((RexCall) e, unknownAs);
+ case NOT:
+ return simplifyNot((RexCall) e, unknownAs);
+ case CASE:
+ return simplifyCase((RexCall) e, unknownAs);
+ case COALESCE:
+ return simplifyCoalesce((RexCall) e);
+ case CAST:
+ case SAFE_CAST:
+ return simplifyCast((RexCall) e);
+ case CEIL:
+ case FLOOR:
+ return simplifyCeilFloor((RexCall) e);
+ case TRIM:
+ return simplifyTrim((RexCall) e);
+ case IS_NULL:
+ case IS_NOT_NULL:
+ case IS_TRUE:
+ case IS_NOT_TRUE:
+ case IS_FALSE:
+ case IS_NOT_FALSE:
+ assert e instanceof RexCall;
+ return simplifyIs((RexCall) e, unknownAs);
+ case EQUALS:
+ case GREATER_THAN:
+ case GREATER_THAN_OR_EQUAL:
+ case LESS_THAN:
+ case LESS_THAN_OR_EQUAL:
+ case NOT_EQUALS:
+ return simplifyComparison((RexCall) e, unknownAs);
+ case SEARCH:
+ return simplifySearch((RexCall) e, unknownAs);
+ case LIKE:
+ return simplifyLike((RexCall) e, unknownAs);
+ case MINUS_PREFIX:
+ case CHECKED_MINUS_PREFIX:
+ return simplifyUnaryMinus((RexCall) e, unknownAs);
+ case PLUS_PREFIX:
+ return simplifyUnaryPlus((RexCall) e, unknownAs);
+ case PLUS:
+ case MINUS:
+ case TIMES:
+ case DIVIDE:
+ case CHECKED_PLUS:
+ case CHECKED_MINUS:
+ case CHECKED_TIMES:
+ case CHECKED_DIVIDE:
+ return simplifyArithmetic((RexCall) e);
+ case M2V:
+ return simplifyM2v((RexCall) e);
+ default:
+ if (e.getClass() == RexCall.class) {
+ return simplifyGenericNode((RexCall) e);
+ } else {
+ return e;
+ }
+ }
+ }
+
+ /** Applies NOT to an expression. */
+ RexNode not(RexNode e) {
+ return RexUtil.not(rexBuilder, e);
+ }
+
+ /** Applies IS NOT FALSE to an expression. */
+ RexNode isNotFalse(RexNode e) {
+ return e.isAlwaysTrue()
+ ? rexBuilder.makeLiteral(true)
+ : e.isAlwaysFalse()
+ ? rexBuilder.makeLiteral(false)
+ : e.getKind() == SqlKind.NOT
+ ? isNotTrue(((RexCall) e).operands.get(0))
+ : predicates.isEffectivelyNotNull(e)
+ ? e // would "CAST(e AS BOOLEAN NOT NULL)" better?
+ : rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_FALSE, e);
+ }
+
+ /** Applies IS NOT TRUE to an expression. */
+ RexNode isNotTrue(RexNode e) {
+ return e.isAlwaysTrue()
+ ? rexBuilder.makeLiteral(false)
+ : e.isAlwaysFalse()
+ ? rexBuilder.makeLiteral(true)
+ : e.getKind() == SqlKind.NOT
+ ? isNotFalse(((RexCall) e).operands.get(0))
+ : predicates.isEffectivelyNotNull(e)
+ ? not(e)
+ : rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_TRUE, e);
+ }
+
+ /** Applies IS TRUE to an expression. */
+ RexNode isTrue(RexNode e) {
+ return e.isAlwaysTrue()
+ ? rexBuilder.makeLiteral(true)
+ : e.isAlwaysFalse()
+ ? rexBuilder.makeLiteral(false)
+ : e.getKind() == SqlKind.NOT
+ ? isFalse(((RexCall) e).operands.get(0))
+ : predicates.isEffectivelyNotNull(e)
+ ? e // would "CAST(e AS BOOLEAN NOT NULL)" better?
+ : rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, e);
+ }
+
+ /** Applies IS FALSE to an expression. */
+ RexNode isFalse(RexNode e) {
+ return e.isAlwaysTrue()
+ ? rexBuilder.makeLiteral(false)
+ : e.isAlwaysFalse()
+ ? rexBuilder.makeLiteral(true)
+ : e.getKind() == SqlKind.NOT
+ ? isTrue(((RexCall) e).operands.get(0))
+ : predicates.isEffectivelyNotNull(e)
+ ? not(e)
+ : rexBuilder.makeCall(SqlStdOperatorTable.IS_FALSE, e);
+ }
+
+ /** Runs simplification inside a non-specialized node. */
+ private RexNode simplifyGenericNode(RexCall e) {
+ final List<RexNode> operands = new ArrayList<>(e.operands);
+ simplifyList(operands, UNKNOWN);
+ if (e.operands.equals(operands)) {
+ return e;
+ }
+ return rexBuilder.makeCall(e.getParserPosition(), e.getType(), e.getOperator(), operands);
+ }
+
+ /**
+ * Try to find a literal with the given value in the input list. The type of the literal must be
+ * one of the numeric types.
+ */
+ private static int findLiteralIndex(List<RexNode> operands, BigDecimal value) {
+ for (int i = 0; i < operands.size(); i++) {
+ if (operands.get(i).isA(SqlKind.LITERAL)) {
+ Comparable comparable = ((RexLiteral) operands.get(i)).getValue();
+ if (comparable instanceof BigDecimal
+ && value.compareTo((BigDecimal) comparable) == 0) {
+ return i;
+ }
+ }
+ }
+ return -1;
+ }
+
+ private RexNode simplifyArithmetic(RexCall e) {
+ if (e.getType().getSqlTypeName().getFamily() != SqlTypeFamily.NUMERIC
+ || e.getOperands().stream()
+ .anyMatch(
+ o ->
+ e.getType().getSqlTypeName().getFamily()
+ != SqlTypeFamily.NUMERIC)) {
+ // we only support simplifying numeric types.
+ return simplifyGenericNode(e);
+ }
+
+ assert e.getOperands().size() == 2;
+
+ switch (e.getKind()) {
+ // These simplifications are safe for both checked and unchecked arithemtic.
+ case PLUS:
+ case CHECKED_PLUS:
+ return simplifyPlus(e);
+ case MINUS:
+ case CHECKED_MINUS:
+ return simplifyMinus(e);
+ case TIMES:
+ case CHECKED_TIMES:
+ return simplifyMultiply(e);
+ case DIVIDE:
+ case CHECKED_DIVIDE:
+ return simplifyDivide(e);
+ default:
+ throw new IllegalArgumentException(
+ "Unsupported arithmeitc operation " + e.getKind());
+ }
+ }
+
+ private RexNode simplifyPlus(RexCall e) {
+ final int zeroIndex = findLiteralIndex(e.operands, BigDecimal.ZERO);
+ if (zeroIndex >= 0) {
+ // return the other operand.
+ RexNode other = e.getOperands().get((zeroIndex + 1) % 2);
+ return other.getType().equals(e.getType())
+ ? other
+ : rexBuilder.makeCast(e.getParserPosition(), e.getType(), other);
+ }
+ return simplifyGenericNode(e);
+ }
+
+ private RexNode simplifyMinus(RexCall e) {
+ final int zeroIndex = findLiteralIndex(e.operands, BigDecimal.ZERO);
+ if (zeroIndex == 1) {
+ RexNode leftOperand = e.getOperands().get(0);
+ return leftOperand.getType().equals(e.getType())
+ ? leftOperand
+ : rexBuilder.makeCast(e.getParserPosition(), e.getType(), leftOperand);
+ }
+ return simplifyGenericNode(e);
+ }
+
+ private RexNode simplifyMultiply(RexCall e) {
+ final int oneIndex = findLiteralIndex(e.operands, BigDecimal.ONE);
+ if (oneIndex >= 0) {
+ // return the other operand.
+ RexNode other = e.getOperands().get((oneIndex + 1) % 2);
+ return other.getType().equals(e.getType())
+ ? other
+ : rexBuilder.makeCast(e.getParserPosition(), e.getType(), other);
+ }
+ return simplifyGenericNode(e);
+ }
+
+ private RexNode simplifyDivide(RexCall e) {
+ final int oneIndex = findLiteralIndex(e.operands, BigDecimal.ONE);
+ if (oneIndex == 1) {
+ RexNode leftOperand = e.getOperands().get(0);
+ return leftOperand.getType().equals(e.getType())
+ ? leftOperand
+ : rexBuilder.makeCast(e.getParserPosition(), e.getType(), leftOperand);
+ }
+ return simplifyGenericNode(e);
+ }
+
+ private RexNode simplifyLike(RexCall e, RexUnknownAs unknownAs) {
+ if (e.operands.get(1) instanceof RexLiteral) {
+ final RexLiteral literal = (RexLiteral) e.operands.get(1);
+ String likeStr = requireNonNull(literal.getValueAs(String.class));
+ Pattern pattern = Pattern.compile("%+");
+ String value = pattern.matcher(likeStr).replaceAll("%");
+ if ("%".equals(value)) {
+ // "x LIKE '%'" or "x LIKE '%...'" simplifies to "x = x"
+ final RexNode x = e.operands.get(0);
+ return simplify(
+ rexBuilder.makeCall(
+ e.getParserPosition(), SqlStdOperatorTable.EQUALS, x, x),
+ unknownAs);
+ }
+ // simplify "x LIKE '%%\%%a%%%'" to "x LIKE '%\%%a%'", default escape is '\'
+ if (e.operands.size() == 2) {
+ e =
+ (RexCall)
+ rexBuilder.makeCall(
+ e.getParserPosition(),
+ e.getOperator(),
+ e.operands.get(0),
+ rexBuilder.makeLiteral(
+ simplifyLikeString(likeStr, '\\', '%')));
+ }
+ if (e.operands.size() == 3 && e.operands.get(2) instanceof RexLiteral) {
+ final RexLiteral escapeLiteral = (RexLiteral) e.operands.get(2);
+ // FLINK MODIFICATION BEGIN
+ final String escapeStr = requireNonNull(escapeLiteral.getValueAs(String.class));
+ if (escapeStr.length() == 1) {
+ char escape = escapeStr.charAt(0);
+ e =
+ (RexCall)
+ rexBuilder.makeCall(
+ e.getParserPosition(),
+ e.getOperator(),
+ e.operands.get(0),
+ rexBuilder.makeLiteral(
+ simplifyLikeString(likeStr, escape, '%')),
+ escapeLiteral);
+ }
+ // FLINK MODIFICATION END
+ }
+ }
+ return simplifyGenericNode(e);
+ }
+
+ // string 'AA%%__%%AA' simplify to 'AA__%AA'
+ // string with even escapes 'AA\\\\%%__%%AA' simplify to 'AA\\__%AA'
+ // string with odd escapes 'AA\\\\\\%%__%%AA' simplify to 'AA\\\\\\%__%AA'
+ private String simplifyMixedWildcards(String str, char escape) {
+ // FLINK MODIFICATION BEGIN
+ Pattern pattern = getMixedWildCardPattern(escape);
+ // FLINK MODIFICATION END
+ Matcher matcher = pattern.matcher(str);
+ StringBuilder builder = new StringBuilder();
+ int from = 0;
+ while (matcher.find()) {
+ int start = matcher.start();
+ String group = requireNonNull(matcher.group(0));
+ if (start > 0
+ && str.charAt(start - 1) == escape
+ && consecutiveSameCharCountBefore(str, start - 1, escape) % 2 == 1) {
+ builder.append(str, from, start + 1);
+ builder.append(simplifyPercentAndUnderline(group.substring(1)));
+ } else {
+ builder.append(str, from, start);
+ builder.append(simplifyPercentAndUnderline(group));
+ }
+ from = matcher.end();
+ }
+ if (from < str.length()) {
+ builder.append(str.substring(from));
+ }
+ return builder.toString();
+ }
+
+ // FLINK MODIFICATION BEGIN
+ private static Pattern getMixedWildCardPattern(char escape) {
+ switch (escape) {
+ case '%':
+ return Pattern.compile("_+");
+ case '_':
+ return Pattern.compile("%+");
+ default:
+ return Pattern.compile("[_%]+");
+ }
+ }
+
+ // FLINK MODIFICATION END
+
+ // Tool method: count the number of consecutive identical characters before index
+ private int consecutiveSameCharCountBefore(String str, int index, char escape) {
+ int count = 0;
+ while (index >= 0) {
+ if (str.charAt(index) != escape) {
+ break;
+ }
+ count++;
+ index--;
+ }
+ return count;
+ }
+
+ // Tool method: simplified string mixed with '%' and '_'
+ private String simplifyPercentAndUnderline(String str) {
+ StringBuilder builder = new StringBuilder();
+ boolean containsPercent = false;
+ for (int index = 0; index < str.length(); index++) {
+ if (str.charAt(index) == '%') {
+ containsPercent = true;
+ continue;
+ }
+ if (str.charAt(index) == '_') {
+ builder.append('_');
+ }
+ }
+ if (containsPercent) {
+ builder.append('%');
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Simplifies like string with escape. A like '%%#%%A%%' escape '#' should simplify to A like
+ * '%#%%A%' escape '#'.
+ */
+ private String simplifyLikeString(String content, char escape, char wildcard) {
+ int escapeCount = 0;
+ int wildcardCount = 0;
+ StringBuilder builder = new StringBuilder();
+ for (int index = 0; index < content.length(); index++) {
+ char c = content.charAt(index);
+ if (c == escape) {
+ builder.append(c);
+ escapeCount++;
+ wildcardCount = 0;
+ continue;
+ }
+ if (c == wildcard) {
+ if (escapeCount % 2 == 1) {
+ builder.append(wildcard);
+ } else if (wildcardCount == 0) {
+ builder.append(wildcard);
+ wildcardCount++;
+ }
+ escapeCount = 0;
+ continue;
+ }
+ builder.append(c);
+ escapeCount = 0;
+ wildcardCount = 0;
+ }
+ return simplifyMixedWildcards(builder.toString(), escape);
+ }
+
+ // e must be a comparison (=, >, >=, <, <=, !=)
+ private RexNode simplifyComparison(RexCall e, RexUnknownAs unknownAs) {
+ //noinspection unchecked
+ return simplifyComparison(e, unknownAs, Comparable.class);
+ }
+
+ // e must be a comparison (=, >, >=, <, <=, !=)
+ private <C extends Comparable<C>> RexNode simplifyComparison(
+ RexCall e, RexUnknownAs unknownAs, Class<C> clazz) {
+ final List<RexNode> operands = new ArrayList<>(e.operands);
+ // UNKNOWN mode is warranted: false = null
+ simplifyList(operands, UNKNOWN);
+
+ // Simplify "x <op> x"
+ final RexNode o0 = operands.get(0);
+ final RexNode o1 = operands.get(1);
+ if (o0.equals(o1) && RexUtil.isDeterministic(o0)) {
+ RexNode newExpr;
+ switch (e.getKind()) {
+ case EQUALS:
+ case GREATER_THAN_OR_EQUAL:
+ case LESS_THAN_OR_EQUAL:
+ // "x = x" simplifies to "null or x is not null" (similarly <= and >=)
+ newExpr =
+ rexBuilder.makeCall(
+ e.getParserPosition(),
+ SqlStdOperatorTable.OR,
+ rexBuilder.makeNullLiteral(e.getType()),
+ rexBuilder.makeCall(
+ e.getParserPosition(),
+ SqlStdOperatorTable.IS_NOT_NULL,
+ o0));
+ return simplify(newExpr, unknownAs);
+ case NOT_EQUALS:
+ case LESS_THAN:
+ case GREATER_THAN:
+ // "x != x" simplifies to "null and x is null" (similarly < and >)
+ newExpr =
+ rexBuilder.makeCall(
+ e.getParserPosition(),
+ SqlStdOperatorTable.AND,
+ rexBuilder.makeNullLiteral(e.getType()),
+ rexBuilder.makeCall(
+ e.getParserPosition(),
+ SqlStdOperatorTable.IS_NULL,
+ o0));
+ return simplify(newExpr, unknownAs);
+ default:
+ // unknown kind
+ }
+ }
+
+ if (o0.getType().getSqlTypeName() == SqlTypeName.BOOLEAN) {
+ Comparison cmp =
+ Comparison.of(
+ rexBuilder.makeCall(e.getParserPosition(), e.getOperator(), o0, o1),
+ node -> true);
+ if (cmp != null) {
+ if (cmp.literal.isAlwaysTrue()) {
+ switch (cmp.kind) {
+ case GREATER_THAN_OR_EQUAL:
+ case EQUALS: // x=true
+ return cmp.ref;
+ case LESS_THAN:
+ case NOT_EQUALS: // x!=true
+ return simplify(not(cmp.ref), unknownAs);
+ case GREATER_THAN:
+ /* this is false, but could be null if x is null */
+ if (!cmp.ref.getType().isNullable()) {
+ return rexBuilder.makeLiteral(false);
+ }
+ break;
+ case LESS_THAN_OR_EQUAL:
+ /* this is true, but could be null if x is null */
+ if (!cmp.ref.getType().isNullable()) {
+ return rexBuilder.makeLiteral(true);
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ if (cmp.literal.isAlwaysFalse()) {
+ switch (cmp.kind) {
+ case EQUALS:
+ case LESS_THAN_OR_EQUAL:
+ return simplify(not(cmp.ref), unknownAs);
+ case NOT_EQUALS:
+ case GREATER_THAN:
+ return cmp.ref;
+ case GREATER_THAN_OR_EQUAL:
+ /* this is true, but could be null if x is null */
+ if (!cmp.ref.getType().isNullable()) {
+ return rexBuilder.makeLiteral(true);
+ }
+ break;
+ case LESS_THAN:
+ /* this is false, but could be null if x is null */
+ if (!cmp.ref.getType().isNullable()) {
+ return rexBuilder.makeLiteral(false);
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ }
+
+ // Simplify "<literal1> <op> <literal2>"
+ // For example, "1 = 2" becomes FALSE;
+ // "1 != 1" becomes FALSE;
+ // "1 != NULL" becomes UNKNOWN (or FALSE if unknownAsFalse);
+ // "1 != '1'" is unchanged because the types are not the same.
+ if (o0.isA(SqlKind.LITERAL)
+ && o1.isA(SqlKind.LITERAL)
+ && SqlTypeUtil.equalSansNullability(
+ rexBuilder.getTypeFactory(), o0.getType(), o1.getType())) {
+ final C v0 = ((RexLiteral) o0).getValueAs(clazz);
+ final C v1 = ((RexLiteral) o1).getValueAs(clazz);
+ if (v0 == null || v1 == null) {
+ return unknownAs == FALSE
+ ? rexBuilder.makeLiteral(false)
+ : rexBuilder.makeNullLiteral(e.getType());
+ }
+ final int comparisonResult = v0.compareTo(v1);
+ switch (e.getKind()) {
+ case EQUALS:
+ return rexBuilder.makeLiteral(comparisonResult == 0);
+ case GREATER_THAN:
+ return rexBuilder.makeLiteral(comparisonResult > 0);
+ case GREATER_THAN_OR_EQUAL:
+ return rexBuilder.makeLiteral(comparisonResult >= 0);
+ case LESS_THAN:
+ return rexBuilder.makeLiteral(comparisonResult < 0);
+ case LESS_THAN_OR_EQUAL:
+ return rexBuilder.makeLiteral(comparisonResult <= 0);
+ case NOT_EQUALS:
+ return rexBuilder.makeLiteral(comparisonResult != 0);
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ RexNode node = simplifyComparisonWithNull(e, unknownAs);
+ if (node instanceof RexLiteral) {
+ return node;
+ }
+
+ // If none of the arguments were simplified, return the call unchanged.
+ final RexNode e2;
+ if (operands.equals(e.operands)) {
+ e2 = e;
+ } else {
+ e2 = rexBuilder.makeCall(e.getParserPosition(), e.op, operands);
+ }
+ return simplifyUsingPredicates(e2, clazz);
+ }
+
+ /**
+ * If this RexNode is a comparison against NULL, return FALSE, otherwise return it unchanged.
+ */
+ static RexNode simplifyComparisonWithNull(
+ RexNode e, RexBuilder rexBuilder, RexUnknownAs unknownAs) {
+ final RexSimplify.Comparison comparison = RexSimplify.Comparison.of(e);
+ if (comparison != null) {
+ boolean againstNull = comparison.literal.isNull();
+ // There is another possibility to check: in a comparison like 1 = null,
+ // the "non-literal" side of the Comparison can be null
+ if (comparison.ref instanceof RexLiteral) {
+ againstNull = againstNull || ((RexLiteral) comparison.ref).isNull();
+ }
+ if (againstNull) {
+ return unknownAs == FALSE
+ ? rexBuilder.makeLiteral(false)
+ : rexBuilder.makeNullLiteral(e.getType());
+ }
+ }
+ return e;
+ }
+
+ public static RexNode simplifyComparisonWithNull(RexNode e, RexBuilder rexBuilder) {
+ return RexSimplify.simplifyComparisonWithNull(e, rexBuilder, FALSE);
+ }
+
+ /**
+ * If this RexNode is a comparison against NULL, return a simplified form, otherwise return it
+ * unchanged.
+ */
+ public RexNode simplifyComparisonWithNull(RexNode e, RexUnknownAs unknownAs) {
+ return simplifyComparisonWithNull(e, this.rexBuilder, unknownAs);
+ }
+
+ /** Simplifies a conjunction of boolean expressions. */
+ @Deprecated // to be removed before 2.0
+ public RexNode simplifyAnds(Iterable<? extends RexNode> nodes) {
+ ensureParanoidOff();
+ return simplifyAnds(nodes, defaultUnknownAs);
+ }
+
+ // package-protected only for a deprecated method; treat as private
+ RexNode simplifyAnds(Iterable<? extends RexNode> nodes, RexUnknownAs unknownAs) {
+ final List<RexNode> terms = new ArrayList<>();
+ final List<RexNode> notTerms = new ArrayList<>();
+ for (RexNode e : nodes) {
+ RelOptUtil.decomposeConjunction(e, terms, notTerms);
+ }
+ simplifyList(terms, UNKNOWN);
+ simplifyList(notTerms, UNKNOWN);
+ if (unknownAs == FALSE) {
+ return simplifyAnd2ForUnknownAsFalse(terms, notTerms);
+ }
+ return simplifyAnd2(terms, notTerms);
+ }
+
+ private void simplifyList(List<RexNode> terms, RexUnknownAs unknownAs) {
+ terms.replaceAll(e -> simplify(e, unknownAs));
+ }
+
+ private void simplifyAndTerms(List<RexNode> terms, RexUnknownAs unknownAs) {
+ RexSimplify simplify = this;
+ for (int i = 0; i < terms.size(); i++) {
+ RexNode t = terms.get(i);
+ if (Predicate.of(t) == null) {
+ continue;
+ }
+ terms.set(i, simplify.simplify(t, unknownAs));
+ RelOptPredicateList newPredicates =
+ simplify.predicates.union(
+ rexBuilder,
+ RelOptPredicateList.of(rexBuilder, terms.subList(i, i + 1)));
+ simplify = simplify.withPredicates(newPredicates);
+ }
+ for (int i = 0; i < terms.size(); i++) {
+ RexNode t = terms.get(i);
+ if (Predicate.of(t) != null) {
+ continue;
+ }
+ terms.set(i, simplify.simplify(t, unknownAs));
+ }
+ }
+
+ private void simplifyOrTerms(List<RexNode> terms, RexUnknownAs unknownAs) {
+ // Suppose we are processing "e1(x) OR e2(x) OR e3(x)". When we are
+ // visiting "e3(x)" we know both "e1(x)" and "e2(x)" are not true (they
+ // may be unknown), because if either of them were true we would have
+ // stopped.
+ RexSimplify simplify = this;
+
+ // 'doneTerms' prevents us from visiting a term in both first and second
+ // loops. If we did this, the second visit would have a predicate saying
+ // that 'term' is false. Effectively, we sort terms: visiting
+ // 'allowedAsPredicate' terms in the first loop, and
+ // non-'allowedAsPredicate' in the second. Each term is visited once.
+ final BitSet doneTerms = new BitSet();
+ for (int i = 0; i < terms.size(); i++) {
+ final RexNode t = terms.get(i);
+ if (!simplify.allowedAsPredicateDuringOrSimplification(t)) {
+ continue;
+ }
+ doneTerms.set(i);
+ final RexNode t2 = simplify.simplify(t, unknownAs);
+ terms.set(i, t2);
+ final RexNode inverse = simplify.simplify(isNotTrue(t2), RexUnknownAs.UNKNOWN);
+ final RelOptPredicateList newPredicates =
+ simplify.predicates.union(
+ rexBuilder,
+ RelOptPredicateList.of(rexBuilder, ImmutableList.of(inverse)));
+ simplify = simplify.withPredicates(newPredicates);
+ }
+ for (int i = 0; i < terms.size(); i++) {
+ final RexNode t = terms.get(i);
+ if (doneTerms.get(i)) {
+ continue; // we visited this term in the first loop
+ }
+ terms.set(i, simplify.simplify(t, unknownAs));
+ }
+ }
+
+ /**
+ * Decides whether the given node could be used as a predicate during the simplification of
+ * other OR operands.
+ */
+ private boolean allowedAsPredicateDuringOrSimplification(final RexNode t) {
+ Predicate predicate = Predicate.of(t);
+ return predicate != null && predicate.allowedInOr(predicates);
+ }
+
+ private RexNode simplifyNot(RexCall call, RexUnknownAs unknownAs) {
+ final RexNode a = call.getOperands().get(0);
+ final List<RexNode> newOperands;
+ switch (a.getKind()) {
+ case NOT:
+ // NOT NOT x ==> x
+ return simplify(((RexCall) a).getOperands().get(0), unknownAs);
+
+ case SEARCH:
+ // NOT SEARCH(x, Sarg[(-inf, 10) OR NULL) ==> SEARCH(x, Sarg[[10, +inf)])
+ final RexCall call2 = (RexCall) a;
+ final RexNode ref = call2.operands.get(0);
+ final RexLiteral literal = (RexLiteral) call2.operands.get(1);
+ final Sarg sarg = literal.getValueAs(Sarg.class);
+ return simplifySearch(
+ call2.clone(
+ call2.type,
+ ImmutableList.of(
+ ref,
+ rexBuilder.makeLiteral(
+ requireNonNull(sarg, "sarg").negate(),
+ literal.getType(),
+ literal.getTypeName()))),
+ unknownAs.negate());
+
+ case LITERAL:
+ if (a.getType().getSqlTypeName() == SqlTypeName.BOOLEAN
+ && !RexLiteral.isNullLiteral(a)) {
+ return rexBuilder.makeLiteral(!RexLiteral.booleanValue(a));
+ }
+ break;
+
+ case AND:
+ // NOT distributivity for AND
+ newOperands = new ArrayList<>();
+ for (RexNode operand : ((RexCall) a).getOperands()) {
+ newOperands.add(simplify(not(operand), unknownAs));
+ }
+ return simplify(
+ rexBuilder.makeCall(
+ call.getParserPosition(), SqlStdOperatorTable.OR, newOperands),
+ unknownAs);
+
+ case OR:
+ // NOT distributivity for OR
+ newOperands = new ArrayList<>();
+ for (RexNode operand : ((RexCall) a).getOperands()) {
+ newOperands.add(simplify(not(operand), unknownAs));
+ }
+ return simplify(
+ rexBuilder.makeCall(
+ call.getParserPosition(), SqlStdOperatorTable.AND, newOperands),
+ unknownAs);
+
+ case CASE:
+ newOperands = new ArrayList<>();
+ List<RexNode> operands = ((RexCall) a).getOperands();
+ for (int i = 0; i < operands.size(); i += 2) {
+ if (i + 1 == operands.size()) {
+ newOperands.add(not(operands.get(i)));
+ } else {
+ newOperands.add(operands.get(i));
+ newOperands.add(not(operands.get(i + 1)));
+ }
+ }
+ return simplify(
+ rexBuilder.makeCall(
+ call.getParserPosition(), SqlStdOperatorTable.CASE, newOperands),
+ unknownAs);
+
+ case IN:
+ case NOT_IN:
+ // do not try to negate
+ break;
+
+ default:
+ final SqlKind negateKind = a.getKind().negate();
+ if (a.getKind() != negateKind) {
+ return simplify(
+ rexBuilder.makeCall(
+ call.getParserPosition(),
+ RexUtil.op(negateKind),
+ ((RexCall) a).getOperands()),
+ unknownAs);
+ }
+ final SqlKind negateKind2 = a.getKind().negateNullSafe();
+ if (a.getKind() != negateKind2) {
+ return simplify(
+ rexBuilder.makeCall(
+ call.getParserPosition(),
+ RexUtil.op(negateKind2),
+ ((RexCall) a).getOperands()),
+ unknownAs);
+ }
+ }
+
+ RexNode a2 = simplify(a, unknownAs.negate());
+ if (a == a2) {
+ return call;
+ }
+ return not(a2);
+ }
+
+ private RexNode simplifyUnaryMinus(RexCall call, RexUnknownAs unknownAs) {
+ final RexNode a = call.getOperands().get(0);
+ if (a.getKind() == SqlKind.MINUS_PREFIX) {
+ // -(-(x)) ==> x
+ return simplify(((RexCall) a).getOperands().get(0), unknownAs);
+ }
+ return simplifyGenericNode(call);
+ }
+
+ private RexNode simplifyUnaryPlus(RexCall call, RexUnknownAs unknownAs) {
+ return simplify(call.getOperands().get(0), unknownAs);
+ }
+
+ private RexNode simplifyIs(RexCall call, RexUnknownAs unknownAs) {
+ final SqlKind kind = call.getKind();
+ final RexNode a = call.getOperands().get(0);
+ final RexNode simplified = simplifyIs1(kind, a, unknownAs);
+ return simplified == null ? call : simplified;
+ }
+
+ private @Nullable RexNode simplifyIs1(SqlKind kind, RexNode a, RexUnknownAs unknownAs) {
+ // UnknownAs.FALSE corresponds to x IS TRUE evaluation
+ // UnknownAs.TRUE to x IS NOT FALSE
+ // Note that both UnknownAs.TRUE and UnknownAs.FALSE only changes the meaning of Unknown
+ // (1) if we are already in UnknownAs.FALSE mode; x IS TRUE can be simplified to x
+ // (2) similarly in UnknownAs.TRUE mode; x IS NOT FALSE can be simplified to x
+ // (3) x IS FALSE could be rewritten to (NOT x) IS TRUE and from there the 1. rule applies
+ // (4) x IS NOT TRUE can be rewritten to (NOT x) IS NOT FALSE and from there the 2. rule
+ // applies
+ if (kind == SqlKind.IS_TRUE && unknownAs == RexUnknownAs.FALSE) {
+ return simplify(a, unknownAs);
+ }
+ if (kind == SqlKind.IS_FALSE && unknownAs == RexUnknownAs.FALSE) {
+ return simplify(not(a), unknownAs);
+ }
+ if (kind == SqlKind.IS_NOT_FALSE && unknownAs == RexUnknownAs.TRUE) {
+ return simplify(a, unknownAs);
+ }
+ if (kind == SqlKind.IS_NOT_TRUE && unknownAs == RexUnknownAs.TRUE) {
+ return simplify(not(a), unknownAs);
+ }
+ final RexNode pred = simplifyIsPredicate(kind, a);
+ if (pred != null) {
+ return pred;
+ }
+
+ return simplifyIs2(kind, a, unknownAs);
+ }
+
+ private @Nullable RexNode simplifyIsPredicate(SqlKind kind, RexNode a) {
+ if (!(RexUtil.isReferenceOrAccess(a, true) || RexUtil.isDeterministic(a))) {
+ return null;
+ }
+
+ for (RexNode p : predicates.pulledUpPredicates) {
+ IsPredicate pred = IsPredicate.of(p);
+ if (pred == null || !a.equals(pred.ref)) {
+ continue;
+ }
+ if (kind == pred.kind) {
+ return rexBuilder.makeLiteral(true);
+ }
+ }
+ return null;
+ }
+
+ private @Nullable RexNode simplifyIs2(SqlKind kind, RexNode a, RexUnknownAs unknownAs) {
+ final RexNode simplified;
+ switch (kind) {
+ case IS_NULL:
+ // x IS NULL ==> FALSE (if x is not nullable)
+ validateStrongPolicy(a);
+ simplified = simplifyIsNull(a);
+ if (simplified != null) {
+ return simplified;
+ }
+ break;
+ case IS_NOT_NULL:
+ // x IS NOT NULL ==> TRUE (if x is not nullable)
+ validateStrongPolicy(a);
+ simplified = simplifyIsNotNull(a);
+ if (simplified != null) {
+ return simplified;
+ }
+ break;
+
+ case IS_TRUE:
+ // x IS TRUE ==> x (if x is not nullable)
+ if (predicates.isEffectivelyNotNull(a)) {
+ return simplify(a, unknownAs);
+ }
+ simplified = simplify(a, RexUnknownAs.FALSE);
+ if (simplified == a) {
+ return null;
+ }
+ return isTrue(simplified);
+
+ case IS_NOT_FALSE:
+ // x IS NOT FALSE ==> x (if x is not nullable)
+ if (predicates.isEffectivelyNotNull(a)) {
+ return simplify(a, unknownAs);
+ }
+ simplified = simplify(a, RexUnknownAs.TRUE);
+ if (simplified == a) {
+ return null;
+ }
+ return isNotFalse(simplified);
+
+ case IS_FALSE:
+ case IS_NOT_TRUE:
+ // x IS NOT TRUE ==> NOT x (if x is not nullable)
+ // x IS FALSE ==> NOT x (if x is not nullable)
+ if (predicates.isEffectivelyNotNull(a)) {
+ return simplify(not(a), unknownAs);
+ }
+ break;
+
+ default:
+ break;
+ }
+ switch (a.getKind()) {
+ case NOT:
+ // (NOT x) IS TRUE ==> x IS FALSE
+ // Similarly for IS NOT TRUE, IS FALSE, etc.
+ //
+ // Note that
+ // (NOT x) IS TRUE !=> x IS FALSE
+ // because of null values.
+ final SqlOperator notKind = RexUtil.op(kind.negateNullSafe());
+ final RexNode arg = ((RexCall) a).operands.get(0);
+ return simplify(rexBuilder.makeCall(notKind, arg), UNKNOWN);
+ default:
+ break;
+ }
+ final RexNode a2 = simplify(a, UNKNOWN);
+ if (a != a2) {
+ return rexBuilder.makeCall(RexUtil.op(kind), ImmutableList.of(a2));
+ }
+ return null; // cannot be simplified
+ }
+
+ private @Nullable RexNode simplifyIsNotNull(RexNode a) {
+ // Simplify the argument first,
+ // call ourselves recursively to see whether we can make more progress.
+ // For example, given
+ // "(CASE WHEN FALSE THEN 1 ELSE 2) IS NOT NULL" we first simplify the
+ // argument to "2", and only then we can simplify "2 IS NOT NULL" to "TRUE".
+ a = simplify(a, UNKNOWN);
+ if (!a.getType().isNullable() && isSafeExpression(a)) {
+ return rexBuilder.makeLiteral(true);
+ }
+ if (RexUtil.isLosslessCast(a)) {
+ if (!a.getType().isNullable()) {
+ return rexBuilder.makeLiteral(true);
+ }
+ return rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, RexUtil.removeCast(a));
+ }
+ if (predicates.pulledUpPredicates.contains(a)) {
+ return rexBuilder.makeLiteral(true);
+ }
+ if (hasCustomNullabilityRules(a.getKind())) {
+ return null;
+ }
+ switch (Strong.policy(a)) {
+ case NOT_NULL:
+ return rexBuilder.makeLiteral(true);
+ case ANY:
+ // "f" is a strong operator, so "f(operand0, operand1) IS NOT NULL"
+ // simplifies to "operand0 IS NOT NULL AND operand1 IS NOT NULL"
+ final List<RexNode> operands = new ArrayList<>();
+ for (RexNode operand : ((RexCall) a).getOperands()) {
+ final RexNode simplified = simplifyIsNotNull(operand);
+ if (simplified == null) {
+ operands.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, operand));
+ } else if (simplified.isAlwaysFalse()) {
+ return rexBuilder.makeLiteral(false);
+ } else {
+ operands.add(simplified);
+ }
+ }
+ return RexUtil.composeConjunction(rexBuilder, operands);
+ case CUSTOM:
+ switch (a.getKind()) {
+ case LITERAL:
+ return rexBuilder.makeLiteral(!((RexLiteral) a).isNull());
+ default:
+ throw new AssertionError(
+ "every CUSTOM policy needs a handler, " + a.getKind());
+ }
+ case AS_IS:
+ default:
+ return null;
+ }
+ }
+
+ private @Nullable RexNode simplifyIsNull(RexNode a) {
+ // Simplify the argument first,
+ // call ourselves recursively to see whether we can make more progress.
+ // For example, given
+ // "(CASE WHEN FALSE THEN 1 ELSE 2) IS NULL" we first simplify the
+ // argument to "2", and only then we can simplify "2 IS NULL" to "FALSE".
+ a = simplify(a, UNKNOWN);
+ if (!a.getType().isNullable() && isSafeExpression(a)) {
+ return rexBuilder.makeLiteral(false);
+ }
+ if (RexUtil.isLosslessCast(a)) {
+ if (!a.getType().isNullable()) {
+ return rexBuilder.makeLiteral(false);
+ }
+ return rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, RexUtil.removeCast(a));
+ }
+ if (RexUtil.isNull(a)) {
+ return rexBuilder.makeLiteral(true);
+ }
+ if (hasCustomNullabilityRules(a.getKind())) {
+ return null;
+ }
+ switch (Strong.policy(a)) {
+ case NOT_NULL:
+ return rexBuilder.makeLiteral(false);
+ case ANY:
+ // "f" is a strong operator, so "f(operand0, operand1) IS NULL" simplifies
+ // to "operand0 IS NULL OR operand1 IS NULL"
+ final List<RexNode> operands = new ArrayList<>();
+ for (RexNode operand : ((RexCall) a).getOperands()) {
+ final RexNode simplified = simplifyIsNull(operand);
+ if (simplified == null) {
+ operands.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, operand));
+ } else {
+ operands.add(simplified);
+ }
+ }
+ return RexUtil.composeDisjunction(rexBuilder, operands, false);
+ case AS_IS:
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Validates strong policy for specified {@link RexNode}.
+ *
+ * @param rexNode Rex node to validate the strong policy
+ * @throws AssertionError If the validation fails
+ */
+ private static void validateStrongPolicy(RexNode rexNode) {
+ if (hasCustomNullabilityRules(rexNode.getKind())) {
+ return;
+ }
+ switch (Strong.policy(rexNode)) {
+ case NOT_NULL:
+ assert !rexNode.getType().isNullable();
+ break;
+ case ANY:
+ List<RexNode> operands = ((RexCall) rexNode).getOperands();
+ if (rexNode.getType().isNullable()) {
+ assert operands.stream()
+ .map(RexNode::getType)
+ .anyMatch(RelDataType::isNullable);
+ } else {
+ assert operands.stream()
+ .map(RexNode::getType)
+ .noneMatch(RelDataType::isNullable);
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ /**
+ * Returns {@code true} if specified {@link SqlKind} has custom nullability rules which depend
+ * not only on the nullability of input operands.
+ *
+ * <p>For example, CAST may be used to change the nullability of its operand type, so it may be
+ * nullable, though the argument type was non-nullable.
+ *
+ * @param sqlKind Sql kind to check
+ * @return {@code true} if specified {@link SqlKind} has custom nullability rules
+ */
+ private static boolean hasCustomNullabilityRules(SqlKind sqlKind) {
+ switch (sqlKind) {
+ case CAST:
+ case ITEM:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ private RexNode simplifyCoalesce(RexCall call) {
+ final Set<RexNode> operandSet = new HashSet<>();
+ final List<RexNode> operands = new ArrayList<>();
+ for (RexNode operand : call.getOperands()) {
+ operand = simplify(operand, UNKNOWN);
+ if (!RexUtil.isNull(operand) && operandSet.add(operand)) {
+ operands.add(operand);
+ }
+ if (!operand.getType().isNullable()) {
+ break;
+ }
+ }
+ switch (operands.size()) {
+ case 0:
+ return rexBuilder.makeNullLiteral(call.type);
+ case 1:
+ return operands.get(0);
+ default:
+ if (operands.equals(call.operands)) {
+ return call;
+ }
+ return call.clone(call.type, operands);
+ }
+ }
+
+ private RexNode simplifyCase(RexCall call, RexUnknownAs unknownAs) {
+ List<CaseBranch> inputBranches =
+ CaseBranch.fromCaseOperands(rexBuilder, new ArrayList<>(call.getOperands()));
+
+ // run simplification on all operands
+ RexSimplify condSimplifier = this.withPredicates(RelOptPredicateList.EMPTY);
+ RexSimplify valueSimplifier = this;
+ RelDataType caseType = call.getType();
+
+ boolean conditionNeedsSimplify = false;
+ CaseBranch lastBranch = null;
+ List<CaseBranch> branches = new ArrayList<>();
+ for (CaseBranch inputBranch : inputBranches) {
+ // simplify the condition
+ RexNode newCond = condSimplifier.simplify(inputBranch.cond, RexUnknownAs.FALSE);
+ if (newCond.isAlwaysFalse()) {
+ // If the condition is false, we do not need to add it
+ continue;
+ }
+
+ // simplify the value
+ RexNode newValue = valueSimplifier.simplify(inputBranch.value, unknownAs);
+
+ // create new branch
+ if (lastBranch != null) {
+ if (lastBranch.value.equals(newValue) && isSafeExpression(newCond)) {
+ // in this case, last branch and new branch have the same conclusion,
+ // hence we create a new composite condition and we do not add it to
+ // the final branches for the time being
+ newCond = rexBuilder.makeCall(SqlStdOperatorTable.OR, lastBranch.cond, newCond);
+ conditionNeedsSimplify = true;
+ } else {
+ // if we reach here, the new branch is not mergeable with the last one,
+ // hence we are going to add the last branch to the final branches.
+ // if the last branch was merged, then we will simplify it first.
+ // otherwise, we just add it
+ CaseBranch branch =
+ generateBranch(conditionNeedsSimplify, condSimplifier, lastBranch);
+ if (!branch.cond.isAlwaysFalse()) {
+ // If the condition is not false, we add it to the final result
+ branches.add(branch);
+ if (branch.cond.isAlwaysTrue()) {
+ // If the condition is always true, we are done
+ lastBranch = null;
+ break;
+ }
+ }
+ conditionNeedsSimplify = false;
+ }
+ }
+ lastBranch = new CaseBranch(newCond, newValue);
+
+ if (newCond.isAlwaysTrue()) {
+ // If the condition is always true, we are done (useful in first loop iteration)
+ break;
+ }
+ }
+ if (lastBranch != null) {
+ // we need to add the last pending branch once we have finished
+ // with the for loop
+ CaseBranch branch = generateBranch(conditionNeedsSimplify, condSimplifier, lastBranch);
+ if (!branch.cond.isAlwaysFalse()) {
+ branches.add(branch);
+ }
+ }
+
+ if (branches.size() == 1) {
+ // we can just return the value in this case (matching the case type)
+ final RexNode value = branches.get(0).value;
+ if (sameTypeOrNarrowsNullability(caseType, value.getType())) {
+ return value;
+ } else {
+ return rexBuilder.makeAbstractCast(
+ call.getParserPosition(), caseType, value, false);
+ }
+ }
+
+ if (call.getType().getSqlTypeName() == SqlTypeName.BOOLEAN) {
+ final RexNode result = simplifyBooleanCase(rexBuilder, branches, unknownAs, caseType);
+ if (result != null) {
+ if (sameTypeOrNarrowsNullability(caseType, result.getType())) {
+ return simplify(result, unknownAs);
+ } else {
+ // If the simplification would widen the nullability
+ RexNode simplified = simplify(result, UNKNOWN);
+ if (!simplified.getType().isNullable()) {
+ return simplified;
+ } else {
+ return rexBuilder.makeCast(
+ call.getParserPosition(), call.getType(), simplified);
+ }
+ }
+ }
+ }
+ List<RexNode> newOperands = CaseBranch.toCaseOperands(branches);
+ if (newOperands.equals(call.getOperands())) {
+ return call;
+ }
+ return rexBuilder.makeCall(call.getParserPosition(), SqlStdOperatorTable.CASE, newOperands);
+ }
+
+ /**
+ * If boolean is true, simplify cond in input branch and return new branch. Otherwise, simply
+ * return input branch.
+ */
+ private static CaseBranch generateBranch(
+ boolean simplifyCond, RexSimplify simplifier, CaseBranch branch) {
+ if (simplifyCond) {
+ // the previous branch was merged, time to simplify it and
+ // add it to the final result
+ return new CaseBranch(
+ simplifier.simplify(branch.cond, RexUnknownAs.FALSE), branch.value);
+ }
+ return branch;
+ }
+
+ /** Return if the new type is the same and at most narrows the nullability. */
+ private boolean sameTypeOrNarrowsNullability(RelDataType oldType, RelDataType newType) {
+ return oldType.equals(newType)
+ || (SqlTypeUtil.equalSansNullability(rexBuilder.typeFactory, oldType, newType)
+ && oldType.isNullable());
+ }
+
+ /** Object to describe a CASE branch. */
+ static final class CaseBranch {
+
+ private final RexNode cond;
+ private final RexNode value;
+
+ CaseBranch(RexNode cond, RexNode value) {
+ this.cond = cond;
+ this.value = value;
+ }
+
+ @Override
+ public String toString() {
+ return cond + " => " + value;
+ }
+
+ /** Given "CASE WHEN p1 THEN v1 ... ELSE e END" returns [(p1, v1), ..., (true, e)]. */
+ private static List<CaseBranch> fromCaseOperands(
+ RexBuilder rexBuilder, List<RexNode> operands) {
+ List<CaseBranch> ret = new ArrayList<>();
+ for (int i = 0; i < operands.size() - 1; i += 2) {
+ ret.add(new CaseBranch(operands.get(i), operands.get(i + 1)));
+ }
+ ret.add(new CaseBranch(rexBuilder.makeLiteral(true), Util.last(operands)));
+ return ret;
+ }
+
+ private static List<RexNode> toCaseOperands(List<CaseBranch> branches) {
+ List<RexNode> ret = new ArrayList<>();
+ for (int i = 0; i < branches.size() - 1; i++) {
+ CaseBranch branch = branches.get(i);
+ ret.add(branch.cond);
+ ret.add(branch.value);
+ }
+ CaseBranch lastBranch = Util.last(branches);
+ assert lastBranch.cond.isAlwaysTrue();
+ ret.add(lastBranch.value);
+ return ret;
+ }
+ }
+
+ /** Decides whether it is safe to flatten the given CASE part into ANDs/ORs. */
+ enum SafeRexVisitor implements RexVisitor<Boolean> {
+ INSTANCE;
+
+ @SuppressWarnings("ImmutableEnumChecker")
+ private final Set<SqlKind> safeOps;
+
+ @SuppressWarnings("ImmutableEnumChecker")
+ private final ImmutableSet<SqlOperator> safeOperators;
+
+ SafeRexVisitor() {
+ ImmutableSet.Builder<SqlOperator> builder = ImmutableSet.builder();
+ builder.addAll(SqlStdOperatorTable.QUANTIFY_OPERATORS);
+ safeOperators = builder.build();
+
+ Set<SqlKind> safeOps = EnumSet.noneOf(SqlKind.class);
+ safeOps.addAll(SqlKind.COMPARISON);
+ safeOps.add(SqlKind.ARRAY_VALUE_CONSTRUCTOR);
+ safeOps.add(SqlKind.PLUS_PREFIX);
+ safeOps.add(SqlKind.MINUS_PREFIX);
+ safeOps.add(SqlKind.CHECKED_MINUS_PREFIX);
+ safeOps.add(SqlKind.PLUS);
+ safeOps.add(SqlKind.MINUS);
+ safeOps.add(SqlKind.TIMES);
+ safeOps.add(SqlKind.CHECKED_PLUS);
+ safeOps.add(SqlKind.CHECKED_MINUS);
+ safeOps.add(SqlKind.CHECKED_TIMES);
+ safeOps.add(SqlKind.IS_FALSE);
+ safeOps.add(SqlKind.IS_NOT_FALSE);
+ safeOps.add(SqlKind.IS_TRUE);
+ safeOps.add(SqlKind.IS_NOT_TRUE);
+ safeOps.add(SqlKind.IS_NULL);
+ safeOps.add(SqlKind.IS_NOT_NULL);
+ safeOps.add(SqlKind.IS_DISTINCT_FROM);
+ safeOps.add(SqlKind.IS_NOT_DISTINCT_FROM);
+ safeOps.add(SqlKind.IN);
+ safeOps.add(SqlKind.SEARCH);
+ safeOps.add(SqlKind.OR);
+ safeOps.add(SqlKind.AND);
+ safeOps.add(SqlKind.NOT);
+ safeOps.add(SqlKind.CASE);
+ safeOps.add(SqlKind.LIKE);
+ safeOps.add(SqlKind.COALESCE);
+ safeOps.add(SqlKind.TRIM);
+ safeOps.add(SqlKind.LTRIM);
+ safeOps.add(SqlKind.RTRIM);
+ safeOps.add(SqlKind.BETWEEN);
+ safeOps.add(SqlKind.CEIL);
+ safeOps.add(SqlKind.FLOOR);
+ safeOps.add(SqlKind.REVERSE);
+ safeOps.add(SqlKind.TIMESTAMP_ADD);
+ safeOps.add(SqlKind.TIMESTAMP_DIFF);
+ this.safeOps = Sets.immutableEnumSet(safeOps);
+ }
+
+ @Override
+ public Boolean visitInputRef(RexInputRef inputRef) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitLocalRef(RexLocalRef localRef) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitLiteral(RexLiteral literal) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitCall(RexCall call) {
+ SqlKind sqlKind = call.getKind();
+ SqlOperator sqlOperator = call.getOperator();
+
+ switch (sqlKind) {
+ case DIVIDE:
+ case MOD:
+ List<RexNode> operands = call.getOperands();
+ boolean isSafe =
+ RexVisitorImpl.visitArrayAnd(this, ImmutableList.of(operands.get(0)));
+ if (!isSafe) {
+ return false;
+ }
+ if (operands.get(1) instanceof RexLiteral) {
+ RexLiteral literal = (RexLiteral) operands.get(1);
+ return RexUtil.isNullLiteral(literal, true);
+ }
+ return false;
+ default:
+ break;
+ }
+
+ if (sqlOperator.isSafeOperator()
+ || RexUtil.isLosslessCast(call)
+ || safeOps.contains(sqlKind)
+ || safeOperators.contains(sqlOperator)) {
+ return RexVisitorImpl.visitArrayAnd(this, call.operands);
+ }
+
+ return false;
+ }
+
+ @Override
+ public Boolean visitOver(RexOver over) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitCorrelVariable(RexCorrelVariable correlVariable) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitDynamicParam(RexDynamicParam dynamicParam) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitRangeRef(RexRangeRef rangeRef) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitFieldAccess(RexFieldAccess fieldAccess) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitSubQuery(RexSubQuery subQuery) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitTableInputRef(RexTableInputRef fieldRef) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitPatternFieldRef(RexPatternFieldRef fieldRef) {
+ return false;
+ }
+
+ @Override
+ public Boolean visitLambda(RexLambda lambda) {
+ return lambda.getExpression().accept(this);
+ }
+
+ @Override
+ public Boolean visitLambdaRef(RexLambdaRef lambdaRef) {
+ return true;
+ }
+
+ @Override
+ public Boolean visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex) {
+ return true;
+ }
+ }
+
+ /**
+ * Analyzes a given {@link RexNode} and decides whenever it is safe to unwind.
+ *
+ * <p>"Safe" means that it only contains a combination of known good operators.
+ *
+ * <p>Division is an unsafe operator; consider the following:
+ *
+ * <pre>case when a > 0 then 1 / a else null end</pre>
+ */
+ static boolean isSafeExpression(RexNode r) {
+ return r.accept(SafeRexVisitor.INSTANCE);
+ }
+
+ private @Nullable RexNode simplifyBooleanCase(
+ RexBuilder rexBuilder,
+ List<CaseBranch> inputBranches,
+ @SuppressWarnings("unused") RexUnknownAs unknownAs,
+ RelDataType branchType) {
+ RexNode result;
+
+ // prepare all condition/branches for boolean interpretation
+ // It's done here make these interpretation changes available to case2or simplifications
+ // but not interfere with the normal simplification recursion
+ List<CaseBranch> branches = new ArrayList<>();
+ for (CaseBranch branch : inputBranches) {
+ if ((!branches.isEmpty() && !isSafeExpression(branch.cond))
+ || !isSafeExpression(branch.value)) {
+ return null;
+ }
+ final RexNode cond = isTrue(branch.cond);
+ final RexNode value;
+ if (!branchType.equals(branch.value.getType())) {
+ value = rexBuilder.makeAbstractCast(branchType, branch.value, false);
+ } else {
+ value = branch.value;
+ }
+ branches.add(new CaseBranch(cond, value));
+ }
+
+ result = simplifyBooleanCaseGeneric(rexBuilder, branches);
+ return result;
+ }
+
+ /**
+ * Generic boolean case simplification.
+ *
+ * <p>Rewrites:
+ *
+ * <pre>
+ * CASE
+ * WHEN p1 THEN x
+ * WHEN p2 THEN y
+ * ELSE z
+ * END
+ * </pre>
+ *
+ * to
+ *
+ * <pre>(p1 and x) or (p2 and y and not(p1)) or (true and z and not(p1) and not(p2))</pre>
+ */
+ private static RexNode simplifyBooleanCaseGeneric(
+ RexBuilder rexBuilder, List<CaseBranch> branches) {
+
+ boolean booleanBranches =
+ branches.stream()
+ .allMatch(
+ branch ->
+ branch.value.isAlwaysTrue()
+ || branch.value.isAlwaysFalse());
+ final List<RexNode> terms = new ArrayList<>();
+ final List<RexNode> notTerms = new ArrayList<>();
+ for (CaseBranch branch : branches) {
+ boolean useBranch = !branch.value.isAlwaysFalse();
+ if (useBranch) {
+ final RexNode branchTerm;
+ if (branch.value.isAlwaysTrue()) {
+ branchTerm = branch.cond;
+ } else {
+ branchTerm =
+ rexBuilder.makeCall(SqlStdOperatorTable.AND, branch.cond, branch.value);
+ }
+ terms.add(RexUtil.andNot(rexBuilder, branchTerm, notTerms));
+ }
+ if (booleanBranches && useBranch) {
+ // we are safe to ignore this branch because for boolean true branches:
+ // a || (b && !a) === a || b
+ } else {
+ notTerms.add(branch.cond);
+ }
+ }
+ return RexUtil.composeDisjunction(rexBuilder, terms);
+ }
+
+ @Deprecated // to be removed before 2.0
+ public RexNode simplifyAnd(RexCall e) {
+ ensureParanoidOff();
+ return simplifyAnd(e, defaultUnknownAs);
+ }
+
+ RexNode simplifyAnd(RexCall e, RexUnknownAs unknownAs) {
+ List<RexNode> operands = RelOptUtil.conjunctions(e);
+
+ if (unknownAs == FALSE && predicateElimination) {
+ simplifyAndTerms(operands, FALSE);
+ } else {
+ simplifyList(operands, unknownAs);
+ }
+
+ final List<RexNode> terms = new ArrayList<>();
+ final List<RexNode> notTerms = new ArrayList<>();
+
+ final SargCollector sargCollector = new SargCollector(rexBuilder, true);
+ operands.forEach(t -> sargCollector.accept(t, terms));
+ if (sargCollector.needToFix()) {
+ operands.clear();
+ terms.forEach(t -> operands.add(SargCollector.fix(rexBuilder, t, unknownAs)));
+ }
+ terms.clear();
+
+ for (RexNode o : operands) {
+ RelOptUtil.decomposeConjunction(o, terms, notTerms);
+ }
+
+ switch (unknownAs) {
+ case FALSE:
+ return simplifyAnd2ForUnknownAsFalse(terms, notTerms, Comparable.class);
+ default:
+ break;
+ }
+ return simplifyAnd2(terms, notTerms);
+ }
+
+ // package-protected only to support a deprecated method; treat as private
+ RexNode simplifyAnd2(List<RexNode> terms, List<RexNode> notTerms) {
+ for (RexNode term : terms) {
+ if (term.isAlwaysFalse()) {
+ return rexBuilder.makeLiteral(false);
+ }
+ }
+ if (terms.isEmpty() && notTerms.isEmpty()) {
+ return rexBuilder.makeLiteral(true);
+ }
+ // If one of the not-disjunctions is a disjunction that is wholly
+ // contained in the disjunctions list, the expression is not
+ // satisfiable.
+ //
+ // Example #1. x AND y AND z AND NOT (x AND y) - not satisfiable
+ // Example #2. x AND y AND NOT (x AND y) - not satisfiable
+ // Example #3. x AND y AND NOT (x AND y AND z) - may be satisfiable
+ List<RexNode> notSatisfiableNullables = null;
+ for (RexNode notDisjunction : notTerms) {
+ final List<RexNode> terms2 = RelOptUtil.conjunctions(notDisjunction);
+ if (!terms.containsAll(terms2)) {
+ // may be satisfiable ==> check other terms
+ continue;
+ }
+ if (!notDisjunction.getType().isNullable()) {
+ // x is NOT nullable, then x AND NOT(x) ==> FALSE
+ return rexBuilder.makeLiteral(false);
+ }
+ // x AND NOT(x) is UNKNOWN for NULL input
+ // So we search for the shortest notDisjunction then convert
+ // original expression to NULL and x IS NULL
+ if (notSatisfiableNullables == null) {
+ notSatisfiableNullables = new ArrayList<>();
+ }
+ notSatisfiableNullables.add(notDisjunction);
+ }
+
+ if (notSatisfiableNullables != null) {
+ // Remove the intersection of "terms" and "notTerms"
+ terms.removeAll(notSatisfiableNullables);
+ notTerms.removeAll(notSatisfiableNullables);
+
+ // The intersection simplify to "null and x1 is null and x2 is null..."
+ terms.add(rexBuilder.makeNullLiteral(notSatisfiableNullables.get(0).getType()));
+ for (RexNode notSatisfiableNullable : notSatisfiableNullables) {
+ terms.add(
+ simplifyIs(
+ (RexCall)
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.IS_NULL,
+ notSatisfiableNullable),
+ UNKNOWN));
+ }
+ }
+ // Add the NOT disjunctions back in.
+ for (RexNode notDisjunction : notTerms) {
+ terms.add(simplify(not(notDisjunction), UNKNOWN));
+ }
+ return RexUtil.composeConjunction(rexBuilder, terms);
+ }
+
+ /**
+ * As {@link #simplifyAnd2(List, List)} but we assume that if the expression returns UNKNOWN it
+ * will be interpreted as FALSE.
+ */
+ RexNode simplifyAnd2ForUnknownAsFalse(List<RexNode> terms, List<RexNode> notTerms) {
+ //noinspection unchecked
+ return simplifyAnd2ForUnknownAsFalse(terms, notTerms, Comparable.class);
+ }
+
+ private <C extends Comparable<C>> RexNode simplifyAnd2ForUnknownAsFalse(
+ List<RexNode> terms, List<RexNode> notTerms, Class<C> clazz) {
+ for (RexNode term : terms) {
+ if (term.isAlwaysFalse() || RexLiteral.isNullLiteral(term)) {
+ return rexBuilder.makeLiteral(false);
+ }
+ }
+ if (terms.isEmpty() && notTerms.isEmpty()) {
+ return rexBuilder.makeLiteral(true);
+ }
+ if (terms.size() == 1 && notTerms.isEmpty()) {
+ // Make sure "x OR y OR x" (a single-term conjunction) gets simplified.
+ return simplify(terms.get(0), FALSE);
+ }
+ // Try to simplify the expression
+ final Multimap<RexNode, Pair<RexNode, RexNode>> equalityTerms = ArrayListMultimap.create();
+ final Map<RexNode, Pair<Range<C>, List<RexNode>>> rangeTerms = new HashMap<>();
+ final Map<RexNode, RexLiteral> equalityConstantTerms = new HashMap<>();
+ final Set<RexNode> negatedTerms = new HashSet<>();
+ final Set<RexNode> nullOperands = new HashSet<>();
+ final Set<RexNode> notNullOperands = new LinkedHashSet<>();
+
+ // Add the predicates from the source to the range terms.
+ for (RexNode predicate : predicates.pulledUpPredicates) {
+ final Comparison comparison = Comparison.of(predicate);
+ if (comparison != null && comparison.kind != SqlKind.NOT_EQUALS) { // not supported yet
+ final C v0 = comparison.literal.getValueAs(clazz);
+ if (v0 != null) {
+ final RexNode result =
+ processRange(
+ rexBuilder,
+ terms,
+ rangeTerms,
+ predicate,
+ comparison.ref,
+ v0,
+ comparison.kind);
+ if (result != null) {
+ // Not satisfiable
+ return result;
+ }
+ }
+ }
+ }
+
+ for (int i = 0; i < terms.size(); i++) {
+ RexNode term = terms.get(i);
+ if (!RexUtil.isDeterministic(term)) {
+ continue;
+ }
+ // Simplify BOOLEAN expressions if possible
+ while (term.getKind() == SqlKind.EQUALS) {
+ RexCall call = (RexCall) term;
+ if (call.getOperands().get(0).isAlwaysTrue()) {
+ term = call.getOperands().get(1);
+ terms.set(i, term);
+ continue;
+ } else if (call.getOperands().get(1).isAlwaysTrue()) {
+ term = call.getOperands().get(0);
+ terms.set(i, term);
+ continue;
+ }
+ break;
+ }
+ switch (term.getKind()) {
+ case EQUALS:
+ case NOT_EQUALS:
+ case LESS_THAN:
+ case GREATER_THAN:
+ case LESS_THAN_OR_EQUAL:
+ case GREATER_THAN_OR_EQUAL:
+ RexCall call = (RexCall) term;
+ final RexNode left = call.getOperands().get(0);
+ final RexNode right = call.getOperands().get(1);
+ final Comparison comparison = Comparison.of(term);
+ // Check for comparison with null values
+ if (comparison != null && comparison.literal.getValue() == null) {
+ return rexBuilder.makeLiteral(false);
+ }
+ // Check for equality on different constants. If the same ref or CAST(ref)
+ // is equal to different constants, this condition cannot be satisfied,
+ // and hence it can be evaluated to FALSE
+ if (term.getKind() == SqlKind.EQUALS) {
+ if (comparison != null) {
+ final RexLiteral literal = comparison.literal;
+ final RexLiteral prevLiteral =
+ equalityConstantTerms.put(comparison.ref, literal);
+
+ if (prevLiteral != null
+ && literal.getType().equals(prevLiteral.getType())
+ && !literal.equals(prevLiteral)) {
+ return rexBuilder.makeLiteral(false);
+ }
+ } else if (RexUtil.isReferenceOrAccess(left, true)
+ && RexUtil.isReferenceOrAccess(right, true)) {
+ equalityTerms.put(left, Pair.of(right, term));
+ }
+ }
+ // Assume the expression a > 5 is part of a Filter condition.
+ // Then we can derive the negated term: a <= 5.
+ // But as the comparison is string based and thus operands order dependent,
+ // we should also add the inverted negated term: 5 >= a.
+ // Observe that for creating the inverted term we invert the list of operands.
+ RexNode negatedTerm = RexUtil.negate(rexBuilder, call);
+ if (negatedTerm != null) {
+ negatedTerms.add(negatedTerm);
+ RexNode invertNegatedTerm =
+ RexUtil.invert(rexBuilder, (RexCall) negatedTerm);
+ if (invertNegatedTerm != null) {
+ negatedTerms.add(invertNegatedTerm);
+ }
+ }
+ // Remove terms that are implied by predicates on the input,
+ // or weaken terms that are partially implied.
+ // E.g. given predicate "x >= 5" and term "x between 3 and 10"
+ // we weaken to term to "x between 5 and 10".
+ final RexNode term2 = simplifyUsingPredicates(term, clazz);
+ if (term2 != term) {
+ terms.set(i, term = term2);
+ }
+ // Range
+ if (comparison != null
+ && comparison.kind != SqlKind.NOT_EQUALS) { // not supported yet
+ final C constant = comparison.literal.getValueAs(clazz);
+ if (constant == null) {
+ break;
+ }
+ final RexNode result =
+ processRange(
+ rexBuilder,
+ terms,
+ rangeTerms,
+ term,
+ comparison.ref,
+ constant,
+ comparison.kind);
+ if (result != null) {
+ // Not satisfiable
+ return result;
+ }
+ }
+ break;
+ case IS_NOT_NULL:
+ notNullOperands.add(((RexCall) term).getOperands().get(0));
+ terms.remove(i);
+ --i;
+ break;
+ case IS_NULL:
+ nullOperands.add(((RexCall) term).getOperands().get(0));
+ break;
+ default:
+ break;
+ }
+ }
+ // Check for equality of two refs wrt equality with constants
+ // Example #1. x=5 AND y=5 AND x=y : x=5 AND y=5
+ // Example #2. x=5 AND y=6 AND x=y - not satisfiable
+ for (RexNode ref1 : equalityTerms.keySet()) {
+ final RexLiteral literal1 = equalityConstantTerms.get(ref1);
+ if (literal1 == null) {
+ continue;
+ }
+ Collection<Pair<RexNode, RexNode>> references = equalityTerms.get(ref1);
+ for (Pair<RexNode, RexNode> ref2 : references) {
+ final RexLiteral literal2 = equalityConstantTerms.get(ref2.left);
+ if (literal2 == null) {
+ continue;
+ }
+ if (literal1.getType().equals(literal2.getType()) && !literal1.equals(literal2)) {
+ // If an expression is equal to two different constants,
+ // it is not satisfiable
+ return rexBuilder.makeLiteral(false);
+ }
+ // Otherwise we can remove the term, as we already know that
+ // the expression is equal to two constants
+ terms.remove(ref2.right);
+ }
+ }
+ // If one of the not-disjunctions is a disjunction that is wholly
+ // contained in the disjunctions list, the expression is not
+ // satisfiable.
+ //
+ // Example #1. x AND y AND z AND NOT (x AND y) - not satisfiable
+ // Example #2. x AND y AND NOT (x AND y) - not satisfiable
+ // Example #3. x AND y AND NOT (x AND y AND z) - may be satisfiable
+ final Set<RexNode> termsSet = new HashSet<>(terms);
+ for (RexNode notDisjunction : notTerms) {
+ if (!RexUtil.isDeterministic(notDisjunction)) {
+ continue;
+ }
+ final List<RexNode> terms2Set = RelOptUtil.conjunctions(notDisjunction);
+ if (termsSet.containsAll(terms2Set)) {
+ return rexBuilder.makeLiteral(false);
+ }
+ }
+ // The negated terms: only deterministic expressions
+ for (RexNode negatedTerm : negatedTerms) {
+ if (termsSet.contains(negatedTerm)) {
+ return rexBuilder.makeLiteral(false);
+ }
+ }
+ // Add the NOT disjunctions back in.
+ for (RexNode notDisjunction : notTerms) {
+ terms.add(not(notDisjunction));
+ }
+ // Find operands that make will let whole expression evaluate to FALSE if set to NULL
+ final Set<RexNode> strongOperands = new HashSet<>();
+ for (RexNode term : terms) {
+ if (!RexUtil.isDeterministic(term)) {
+ continue;
+ }
+ final VariableCollector collector = new VariableCollector();
+ term.accept(collector);
+ for (RexInputRef ref : collector.refs) {
+ final boolean strong = Strong.isNotTrue(term, ImmutableBitSet.of(ref.index));
+ if (strong) {
+ strongOperands.add(ref);
+ }
+ }
+ final RexUtil.FieldAccessFinder fieldAccessFinder = new RexUtil.FieldAccessFinder();
+ term.accept(fieldAccessFinder);
+ for (RexFieldAccess rexFieldAccess : fieldAccessFinder.getFieldAccessList()) {
+ final boolean strong = Strong.of(ImmutableSet.of(rexFieldAccess)).isNotTrue(term);
+ if (strong) {
+ strongOperands.add(rexFieldAccess);
+ }
+ }
+ }
+ // If one column should be null and is in a comparison predicate,
+ // it is not satisfiable.
+ // Example. IS NULL(x) AND x < 5 - not satisfiable
+ if (!Collections.disjoint(nullOperands, strongOperands)) {
+ return rexBuilder.makeLiteral(false);
+ }
+ // Remove not necessary IS NOT NULL expressions.
+ // Example. IS NOT NULL(x) AND x < 5 : x < 5
+ for (RexNode operand : notNullOperands) {
+ if (!strongOperands.contains(operand)) {
+ terms.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, operand));
+ }
+ }
+ return RexUtil.composeConjunction(rexBuilder, terms);
+ }
+
+ private <C extends Comparable<C>> RexNode simplifyUsingPredicates(RexNode e, Class<C> clazz) {
+ if (predicates.pulledUpPredicates.isEmpty()) {
+ return e;
+ }
+
+ final Comparison comparison = Comparison.of(e);
+ // Check for comparison with null values
+ if (comparison == null || comparison.literal.getValue() == null) {
+ return e;
+ }
+
+ final C v0 = comparison.literal.getValueAs(clazz);
+ if (v0 == null) {
+ return e;
+ }
+ final RangeSet<C> rangeSet = rangeSet(comparison.kind, v0);
+ final RangeSet<C> rangeSet2 =
+ residue(comparison.ref, rangeSet, predicates.pulledUpPredicates, clazz);
+ if (rangeSet2.isEmpty()) {
+ // Term is impossible to satisfy given these predicates
+ return rexBuilder.makeLiteral(false);
+ } else if (rangeSet2.equals(rangeSet)) {
+ // no change
+ return e;
+ } else if (rangeSet2.equals(RangeSets.rangeSetAll())) {
+ // Range is always satisfied given these predicates; but nullability might
+ // be problematic
+ return simplify(
+ rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, comparison.ref),
+ RexUnknownAs.UNKNOWN);
+ } else if (rangeSet2.asRanges().size() == 1
+ && Iterables.getOnlyElement(rangeSet2.asRanges()).hasLowerBound()
+ && Iterables.getOnlyElement(rangeSet2.asRanges()).hasUpperBound()
+ && Iterables.getOnlyElement(rangeSet2.asRanges())
+ .lowerEndpoint()
+ .equals(Iterables.getOnlyElement(rangeSet2.asRanges()).upperEndpoint())) {
+ final Range<C> r = Iterables.getOnlyElement(rangeSet2.asRanges());
+ // range is now a point; it's worth simplifying
+ return rexBuilder.makeCall(
+ SqlStdOperatorTable.EQUALS,
+ comparison.ref,
+ rexBuilder.makeLiteral(
+ r.lowerEndpoint(),
+ comparison.literal.getType(),
+ comparison.literal.getTypeName()));
+ } else {
+ // range has been reduced but it's not worth simplifying
+ return e;
+ }
+ }
+
+ /**
+ * Weakens a term so that it checks only what is not implied by predicates.
+ *
+ * <p>The term is broken into "ref comparison constant", for example "$0 < 5".
+ *
+ * <p>Examples:
+ *
+ * <ul>
+ * <li>{@code residue($0 < 10, [$0 < 5])} returns {@code true}
+ * <li>{@code residue($0 < 10, [$0 < 20, $0 > 0])} returns {@code $0 < 10}
+ * </ul>
+ */
+ private static <C extends Comparable<C>> RangeSet<C> residue(
+ RexNode ref, RangeSet<C> r0, List<RexNode> predicates, Class<C> clazz) {
+ RangeSet<C> result = r0;
+ for (RexNode predicate : predicates) {
+ switch (predicate.getKind()) {
+ case EQUALS:
+ case NOT_EQUALS:
+ case LESS_THAN:
+ case LESS_THAN_OR_EQUAL:
+ case GREATER_THAN:
+ case GREATER_THAN_OR_EQUAL:
+ final RexCall call = (RexCall) predicate;
+ final Comparison comparison = Comparison.of(call);
+ if (comparison != null && comparison.ref.equals(ref)) {
+ final C c1 = comparison.literal.getValueAs(clazz);
+ if (c1 == null) {
+ throw new AssertionError(
+ "value must not be null in " + comparison.literal);
+ }
+ switch (predicate.getKind()) {
+ case NOT_EQUALS:
+ // We want to intersect result with the range set of everything but
+ // c1. We subtract the point c1 from result, which is equivalent.
+ final Range<C> pointRange = range(SqlKind.EQUALS, c1);
+ final RangeSet<C> notEqualsRangeSet =
+ ImmutableRangeSet.of(pointRange).complement();
+ if (result.enclosesAll(notEqualsRangeSet)) {
+ result = RangeSets.rangeSetAll();
+ continue;
+ }
+ result = RangeSets.minus(result, pointRange);
+ break;
+ default:
+ final Range<C> r1 = range(comparison.kind, c1);
+ if (result.encloses(r1)) {
+ // Given these predicates, term is always satisfied.
+ // e.g. r0 is "$0 < 10", r1 is "$0 < 5"
+ result = RangeSets.rangeSetAll();
+ continue;
+ }
+ result = result.subRangeSet(r1);
+ }
+ if (result.isEmpty()) {
+ break; // short-cut
+ }
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Simplifies OR(x, x) into x, and similar. The simplified expression returns UNKNOWN values as
+ * is (not as FALSE).
+ */
+ @Deprecated // to be removed before 2.0
+ public RexNode simplifyOr(RexCall call) {
+ ensureParanoidOff();
+ return simplifyOr(call, UNKNOWN);
+ }
+
+ private RexNode simplifyOr(RexCall call, RexUnknownAs unknownAs) {
+ assert call.getKind() == SqlKind.OR;
+ final List<RexNode> terms0 = RelOptUtil.disjunctions(call);
+ final List<RexNode> terms;
+ if (predicateElimination) {
+ terms = Util.moveToHead(terms0, e -> e.getKind() == SqlKind.IS_NULL);
+ simplifyOrTerms(terms, unknownAs);
+ } else {
+ terms = terms0;
+ simplifyList(terms, unknownAs);
+ }
+ return simplifyOrs(terms, unknownAs);
+ }
+
+ /**
+ * Simplifies a list of terms and combines them into an OR. Modifies the list in place. The
+ * simplified expression returns UNKNOWN values as is (not as FALSE).
+ */
+ @Deprecated // to be removed before 2.0
+ public RexNode simplifyOrs(List<RexNode> terms) {
+ ensureParanoidOff();
+ return simplifyOrs(terms, UNKNOWN);
+ }
+
+ private void ensureParanoidOff() {
+ if (paranoid) {
+ throw new UnsupportedOperationException("Paranoid is not supported for this method");
+ }
+ }
+
+ /** Simplifies a list of terms and combines them into an OR. Modifies the list in place. */
+ private RexNode simplifyOrs(List<RexNode> terms, RexUnknownAs unknownAs) {
+ final SargCollector sargCollector = new SargCollector(rexBuilder, false);
+ final List<RexNode> newTerms = new ArrayList<>();
+ terms.forEach(t -> sargCollector.accept(t, newTerms));
+ if (sargCollector.needToFix()) {
+ terms.clear();
+ newTerms.forEach(t -> terms.add(SargCollector.fix(rexBuilder, t, unknownAs)));
+ }
+
+ // CALCITE-3198 Auxiliary map to simplify cases like:
+ // X <> A OR X <> B => X IS NOT NULL or NULL
+ // The map key will be the 'X'; and the value the first call 'X<>A' that is found,
+ // or 'X IS NOT NULL' if a simplification takes place (because another 'X<>B' is found)
+ final Map<RexNode, RexNode> notEqualsComparisonMap = new HashMap<>();
+ final RexLiteral trueLiteral = rexBuilder.makeLiteral(true);
+ for (int i = 0; i < terms.size(); i++) {
+ final RexNode term = terms.get(i);
+ switch (term.getKind()) {
+ case LITERAL:
+ if (RexLiteral.isNullLiteral(term)) {
+ if (unknownAs == FALSE) {
+ terms.remove(i);
+ --i;
+ continue;
+ } else if (unknownAs == TRUE) {
+ return trueLiteral;
+ }
+ } else {
+ if (RexLiteral.booleanValue(term)) {
+ return term; // true
+ } else {
+ terms.remove(i);
+ --i;
+ continue;
+ }
+ }
+ break;
+ case NOT_EQUALS:
+ final Comparison notEqualsComparison = Comparison.of(term);
+ if (notEqualsComparison != null) {
+ // We are dealing with a X<>A term, check if we saw before another
+ // NOT_EQUALS involving X
+ final RexNode prevNotEquals =
+ notEqualsComparisonMap.get(notEqualsComparison.ref);
+ if (prevNotEquals == null) {
+ // This is the first NOT_EQUALS involving X, put it in the map
+ notEqualsComparisonMap.put(notEqualsComparison.ref, term);
+ } else {
+ // There is already in the map another NOT_EQUALS involving X:
+ // - if it is already an IS_NOT_NULL: it was already simplified,
+ // ignore this term
+ // - if it is not an IS_NOT_NULL (i.e. it is a NOT_EQUALS): check
+ // comparison values
+ if (prevNotEquals.getKind() != SqlKind.IS_NOT_NULL) {
+ final Comparable comparable1 =
+ notEqualsComparison.literal.getValue();
+ final Comparable comparable2 =
+ castNonNull(Comparison.of(prevNotEquals))
+ .literal
+ .getValue();
+ //noinspection unchecked
+ if (comparable1 != null
+ && comparable2 != null
+ && comparable1.compareTo(comparable2) != 0) {
+ // X <> A OR X <> B => X IS NOT NULL OR NULL
+ final RexNode isNotNull =
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.IS_NOT_NULL,
+ notEqualsComparison.ref);
+ final RexNode constantNull =
+ rexBuilder.makeNullLiteral(trueLiteral.getType());
+ final RexNode newCondition =
+ simplify(
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.OR,
+ isNotNull,
+ constantNull),
+ unknownAs);
+ if (newCondition.isAlwaysTrue()) {
+ return trueLiteral;
+ }
+ notEqualsComparisonMap.put(notEqualsComparison.ref, isNotNull);
+ final int pos = terms.indexOf(prevNotEquals);
+ terms.set(pos, newCondition);
+ }
+ }
+ terms.remove(i);
+ --i;
+ continue;
+ }
+ }
+ break;
+ case IS_NOT_TRUE:
+ RexNode arg = ((RexCall) term).getOperands().get(0);
+ if (isSafeExpression(arg) && terms.contains(arg)) {
+ return trueLiteral;
+ }
+ break;
+ case NOT:
+ RexNode x = ((RexCall) term).getOperands().get(0);
+ if (isSafeExpression(x) && terms.contains(x)) {
+ if (!x.getType().isNullable()) {
+ return trueLiteral;
+ }
+
+ final RexNode isNotNull =
+ rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, x);
+ terms.set(terms.indexOf(x), simplifyIs((RexCall) isNotNull, unknownAs));
+ terms.set(i, rexBuilder.makeNullLiteral(x.getType()));
+ i--;
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ return RexUtil.composeDisjunction(rexBuilder, terms);
+ }
+
+ private void verify(RexNode before, RexNode simplified, RexUnknownAs unknownAs) {
+ if (simplified.isAlwaysFalse() && before.isAlwaysTrue()) {
+ throw new AssertionError(
+ "always true [" + before + "] simplified to always false [" + simplified + "]");
+ }
+ if (simplified.isAlwaysTrue() && before.isAlwaysFalse()) {
+ throw new AssertionError(
+ "always false [" + before + "] simplified to always true [" + simplified + "]");
+ }
+ final RexAnalyzer foo0 = new RexAnalyzer(before, predicates);
+ final RexAnalyzer foo1 = new RexAnalyzer(simplified, predicates);
+ if (foo0.unsupportedCount > 0 || foo1.unsupportedCount > 0) {
+ // Analyzer cannot handle this expression currently
+ return;
+ }
+ if (!foo0.variables.containsAll(foo1.variables)) {
+ throw new AssertionError(
+ "variable mismatch: "
+ + before
+ + " has "
+ + foo0.variables
+ + ", "
+ + simplified
+ + " has "
+ + foo1.variables);
+ }
+ assignment_loop:
+ for (Map<RexNode, Comparable> map : foo0.assignments()) {
+ for (RexNode predicate : predicates.pulledUpPredicates) {
+ final Comparable v = RexInterpreter.evaluate(predicate, map);
+ if (!Boolean.TRUE.equals(v)) {
+ continue assignment_loop;
+ }
+ }
+ Comparable v0 = RexInterpreter.evaluate(foo0.e, map);
+ if (v0 == null) {
+ throw new AssertionError("interpreter returned null for " + foo0.e);
+ }
+ Comparable v1 = RexInterpreter.evaluate(foo1.e, map);
+ if (v1 == null) {
+ throw new AssertionError("interpreter returned null for " + foo1.e);
+ }
+ if (before.getType().getSqlTypeName() == SqlTypeName.BOOLEAN) {
+ switch (unknownAs) {
+ case FALSE:
+ case TRUE:
+ if (v0 == NullSentinel.INSTANCE) {
+ v0 = unknownAs.toBoolean();
+ }
+ if (v1 == NullSentinel.INSTANCE) {
+ v1 = unknownAs.toBoolean();
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ if (!v0.equals(v1)) {
+ throw new AssertionError(
+ "result mismatch (unknown as "
+ + unknownAs
+ + "): when applied to "
+ + map
+ + ",\n"
+ + before
+ + " yielded "
+ + v0
+ + ";\n"
+ + simplified
+ + " yielded "
+ + v1);
+ }
+ }
+ }
+
+ private RexNode simplifySearch(RexCall call, RexUnknownAs unknownAs) {
+ assert call.getKind() == SqlKind.SEARCH;
+ final RexNode operand = call.getOperands().get(0);
+ final RexNode simplifiedOperand = simplify(operand, unknownAs);
+ final boolean operandUnchanged = operand.equals(simplifiedOperand);
+ final RexNode searchOperand = operandUnchanged ? operand : simplifiedOperand;
+ if (call.getOperands().get(1) instanceof RexLiteral) {
+ RexLiteral literal = (RexLiteral) call.getOperands().get(1);
+ final Sarg sarg = castNonNull(literal.getValueAs(Sarg.class));
+ if (sarg.isAll() || sarg.isNone()) {
+ RexNode rexNode = RexUtil.simpleSarg(rexBuilder, searchOperand, sarg, unknownAs);
+ return simplify(rexNode, unknownAs);
+ }
+ // Remove null from sarg if the left-hand side is never null
+ if (sarg.nullAs != UNKNOWN) {
+ final RexNode simplified = simplifyIs1(SqlKind.IS_NULL, searchOperand, unknownAs);
+ if (simplified != null && simplified.isAlwaysFalse()) {
+ final Sarg sarg2 = Sarg.of(UNKNOWN, sarg.rangeSet);
+ final RexLiteral literal2 =
+ rexBuilder.makeLiteral(sarg2, literal.getType(), literal.getTypeName());
+ // Now we've strengthened the Sarg, try to simplify again
+ return simplifySearch(
+ call.clone(call.type, ImmutableList.of(searchOperand, literal2)),
+ unknownAs);
+ }
+ } else if (sarg.isPoints() && sarg.pointCount <= 1) {
+ // Expand "SEARCH(x, Sarg([point])" to "x = point"
+ // and "SEARCH(x, Sarg([])" to "false"
+ return RexUtil.expandSearch(rexBuilder, null, call);
+ }
+ }
+ return operandUnchanged
+ ? call
+ : call.clone(
+ call.type, ImmutableList.of(simplifiedOperand, call.getOperands().get(1)));
+ }
+
+ private RexNode simplifyCast(RexCall e) {
+ RexNode operand = e.getOperands().get(0);
+ operand = simplify(operand, UNKNOWN);
+ // The type of DYNAMIC_PARAM is indeterminate, so the cast cannot be eliminated
+ if (operand.getKind() != SqlKind.DYNAMIC_PARAM
+ && sameTypeOrNarrowsNullability(e.getType(), operand.getType())
+ // DECIMAL casts are never no-ops: they perform bounds checking
+ && e.getType().getSqlTypeName() != SqlTypeName.DECIMAL) {
+ return operand;
+ }
+ if (RexUtil.isLosslessCast(operand)) {
+ // x :: y below means cast(x as y) (which is PostgreSQL-specific cast by the way)
+ // A) Remove lossless casts:
+ // A.1) intExpr :: bigint :: int => intExpr
+ // A.2) char2Expr :: char(5) :: char(2) => char2Expr
+ // B) There are cases when we can't remove two casts, but we could probably remove inner
+ // one
+ // B.1) char2expression :: char(4) :: char(5) -> char2expression :: char(5)
+ // B.2) char2expression :: char(10) :: char(5) -> char2expression :: char(5)
+ // B.3) char2expression :: varchar(10) :: char(5) -> char2expression :: char(5)
+ // B.4) char6expression :: varchar(10) :: char(5) -> char6expression :: char(5)
+ // C) Simplification is not possible:
+ // C.1) char6expression :: char(3) :: char(5) -> must not be changed
+ // the input is truncated to 3 chars, so we can't use char6expression :: char(5)
+ // C.2) varchar2Expr :: char(5) :: varchar(2) -> must not be changed
+ // the input have to be padded with spaces (up to 2 chars)
+ // C.3) char2expression :: char(4) :: varchar(5) -> must not be changed
+ // would not have the padding
+
+ // The approach seems to be:
+ // 1) Ensure inner cast is lossless (see if above)
+ // 2) If operand of the inner cast has the same type as the outer cast,
+ // remove two casts except C.2 or C.3-like pattern (== inner cast is CHAR)
+ // 3) If outer cast is lossless, remove inner cast (B-like cases)
+
+ // Here we try to remove two casts in one go (A-like cases)
+ RexNode intExpr = ((RexCall) operand).operands.get(0);
+ // intExpr == CHAR detects A.1
+ // operand != CHAR detects C.2
+ if ((intExpr.getType().getSqlTypeName() == SqlTypeName.CHAR
+ || operand.getType().getSqlTypeName() != SqlTypeName.CHAR)
+ && sameTypeOrNarrowsNullability(e.getType(), intExpr.getType())) {
+ return intExpr;
+ }
+ // Here we try to remove inner cast (B-like cases)
+ if (RexUtil.isLosslessCast(intExpr.getType(), operand.getType())
+ && (e.getType().getSqlTypeName() == operand.getType().getSqlTypeName()
+ || e.getType().getSqlTypeName() == SqlTypeName.CHAR
+ || operand.getType().getSqlTypeName() != SqlTypeName.CHAR)
+ && SqlTypeCoercionRule.instance()
+ .canApplyFrom(
+ intExpr.getType().getSqlTypeName(),
+ e.getType().getSqlTypeName())) {
+ return rexBuilder.makeCast(e.getParserPosition(), e.getType(), intExpr);
+ }
+ }
+ final boolean safe = e.getKind() == SqlKind.SAFE_CAST;
+ switch (operand.getKind()) {
+ case LITERAL:
+ final RexLiteral literal = (RexLiteral) operand;
+ final Comparable value = literal.getValueAs(Comparable.class);
+ final SqlTypeName typeName = literal.getTypeName();
+
+ // First, try to remove the cast without changing the value.
+ // makeCast and canRemoveCastFromLiteral have the same logic, so we are
+ // sure to be able to remove the cast.
+ if (rexBuilder.canRemoveCastFromLiteral(e.getType(), value, typeName)) {
+ return rexBuilder.makeCast(e.getParserPosition(), e.getType(), operand);
+ }
+
+ // Next, try to convert the value to a different type,
+ // e.g. CAST('123' as integer)
+ switch (literal.getTypeName()) {
+ case TIME:
+ switch (e.getType().getSqlTypeName()) {
+ case TIMESTAMP:
+ return e;
+ default:
+ break;
+ }
+ break;
+ default:
+ break;
+ }
+ final List<RexNode> reducedValues = new ArrayList<>();
+ final RexNode simplifiedExpr =
+ e.operandCount() == 2
+ ? rexBuilder.makeCast(
+ e.getParserPosition(),
+ e.getType(),
+ operand,
+ safe,
+ safe,
+ (RexLiteral) e.getOperands().get(1))
+ : rexBuilder.makeCast(e.getType(), operand, safe, safe);
+ executor.reduce(rexBuilder, ImmutableList.of(simplifiedExpr), reducedValues);
+ RexNode reducedRexNode = requireNonNull(Iterables.getOnlyElement(reducedValues));
+ if (reducedRexNode.isA(SqlKind.CAST)) {
+ RexNode reducedOperand = ((RexCall) reducedRexNode).getOperands().get(0);
+ if (sameTypeOrNarrowsNullability(
+ reducedRexNode.getType(), reducedOperand.getType())) {
+ return reducedOperand;
+ }
+ }
+ return reducedRexNode;
+ default:
+ if (operand == e.getOperands().get(0)) {
+ return e;
+ } else {
+ return e.operands.size() > 1
+ ? rexBuilder.makeCast(
+ e.getParserPosition(),
+ e.getType(),
+ operand,
+ safe,
+ safe,
+ (RexLiteral) e.getOperands().get(1))
+ : rexBuilder.makeCast(e.getType(), operand, safe, safe);
+ }
+ }
+ }
+
+ /**
+ * Tries to simplify CEIL/FLOOR function on top of CEIL/FLOOR.
+ *
+ * <p>Examples:
+ *
+ * <ul>
+ * <li>{@code floor(floor($0, flag(hour)), flag(day))} returns {@code floor($0, flag(day))}
+ * <li>{@code ceil(ceil($0, flag(second)), flag(day))} returns {@code ceil($0, flag(day))}
+ * <li>{@code floor(floor($0, flag(day)), flag(second))} does not change
+ * </ul>
+ */
+ private RexNode simplifyCeilFloor(RexCall e) {
+ if (e.getOperands().size() != 2) {
+ // Bail out since we only simplify floor <date>
+ return e;
+ }
+ final RexNode operand = simplify(e.getOperands().get(0), UNKNOWN);
+ if (e.getKind() == operand.getKind()) {
+ assert e.getKind() == SqlKind.CEIL || e.getKind() == SqlKind.FLOOR;
+ // CEIL/FLOOR on top of CEIL/FLOOR
+ final RexCall child = (RexCall) operand;
+ if (child.getOperands().size() != 2) {
+ // Bail out since we only simplify ceil/floor <date>
+ return e;
+ }
+ final RexLiteral parentFlag = (RexLiteral) e.operands.get(1);
+ final TimeUnitRange parentFlagValue = (TimeUnitRange) parentFlag.getValue();
+ final RexLiteral childFlag = (RexLiteral) child.operands.get(1);
+ final TimeUnitRange childFlagValue = (TimeUnitRange) childFlag.getValue();
+ if (parentFlagValue != null && childFlagValue != null) {
+ if (canRollUp(parentFlagValue.startUnit, childFlagValue.startUnit)) {
+ return e.clone(
+ e.getType(), ImmutableList.of(child.getOperands().get(0), parentFlag));
+ }
+ }
+ }
+ return e.clone(e.getType(), ImmutableList.of(operand, e.getOperands().get(1)));
+ }
+
+ /**
+ * Simplify TRIM function by eliminating nested duplication.
+ *
+ * <p>Examples:
+ *
+ * <ul>
+ * <li>{@code trim(trim(' aa '))} returns {@code trim(' aa ')}
+ * <li>{@code trim(BOTH ' ' from trim(BOTH ' ' from ' aa '))} returns {@code trim(BOTH ' '
+ * from ' aa ')}
+ * <li>{@code trim(LEADING 'a' from trim(BOTH ' ' from ' aa '))} does not change
+ * </ul>
+ */
+ private RexNode simplifyTrim(RexCall e) {
+ if (e.getOperands().size() != 3) {
+ return e;
+ }
+
+ RexNode trimType = simplify(e.operands.get(0));
+ RexNode trimed = simplify(e.operands.get(1));
+ if (e.getOperands().get(2) instanceof RexCall) {
+ RexCall childNode = (RexCall) e.getOperands().get(2);
+ // only strings with the same trim method and deduplication will be eliminated.
+ if (childNode.getKind() == SqlKind.TRIM
+ && trimType.equals(simplify(childNode.operands.get(0)))
+ && trimed.equals(simplify(childNode.operands.get(1)))) {
+ return simplifyTrim(childNode);
+ }
+ }
+
+ ArrayList<RexNode> rexNodes = new ArrayList<>();
+ rexNodes.add(trimType);
+ rexNodes.add(trimed);
+ rexNodes.add(simplify(e.operands.get(2)));
+ RexNode rexNode = rexBuilder.makeCall(e.getType(), e.getOperator(), rexNodes);
+ return rexNode;
+ }
+
+ /** Method that returns whether we can rollup from inner time unit to outer time unit. */
+ private static boolean canRollUp(TimeUnit outer, TimeUnit inner) {
+ // Special handling for QUARTER as it is not in the expected
+ // order in TimeUnit
+ switch (outer) {
+ case YEAR:
+ case MONTH:
+ case DAY:
+ case HOUR:
+ case MINUTE:
+ case SECOND:
+ case MILLISECOND:
+ case MICROSECOND:
+ switch (inner) {
+ case YEAR:
+ case QUARTER:
+ case MONTH:
+ case DAY:
+ case HOUR:
+ case MINUTE:
+ case SECOND:
+ case MILLISECOND:
+ case MICROSECOND:
+ if (inner == TimeUnit.QUARTER) {
+ return outer == TimeUnit.YEAR;
+ }
+ return outer.ordinal() <= inner.ordinal();
+ default:
+ break;
+ }
+ break;
+ case QUARTER:
+ switch (inner) {
+ case QUARTER:
+ case MONTH:
+ case DAY:
+ case HOUR:
+ case MINUTE:
+ case SECOND:
+ case MILLISECOND:
+ case MICROSECOND:
+ return true;
+ default:
+ break;
+ }
+ break;
+ default:
+ break;
+ }
+ return false;
+ }
+
+ /**
+ * Simplifies a measure being converted immediately (in the same SELECT clause) back to a value.
+ *
+ * <p>For most expressions {@code e}, simplifies "{@code m2v(v2m(e))}" to "{@code e}". For
+ * example, "{@code SELECT deptno + 1 AS MEASURE m}" is equivalent to "{@code SELECT deptno + 1
+ * AS m}".
+ *
+ * <p>The exception is aggregate functions. "{@code SELECT COUNT(*) + 1 AS MEASURE m}"
+ * simplifies to "{@code SELECT COUNT(*) OVER (ROWS CURRENT ROW) + 1 AS MEASURE m}".
+ *
+ * @param e Call to {@code M2V} to be simplified
+ * @return Simplified call
+ */
+ private RexNode simplifyM2v(RexCall e) {
+ assert e.op.kind == SqlKind.M2V;
+ final RexNode operand = e.getOperands().get(0);
+ switch (operand.getKind()) {
+ case V2M:
+ // M2V(V2M(x)) --> x
+ return flattenAggregate(((RexCall) operand).operands.get(0));
+ default:
+ return e;
+ }
+ }
+
+ /**
+ * Traverses over an expression, converting aggregate functions into single-row aggregate
+ * functions.
+ */
+ private RexNode flattenAggregate(RexNode e) {
+ return e.accept(
+ new RexShuttle() {
+ @Override
+ public RexNode visitCall(RexCall call) {
+ if (call.op.isAggregator()) {
+ final RexWindow w =
+ rexBuilder.makeWindow(
+ ImmutableList.of(),
+ ImmutableList.of(),
+ RexWindowBounds.CURRENT_ROW,
+ RexWindowBounds.CURRENT_ROW,
+ true);
+ return new RexOver(
+ call.type,
+ (SqlAggFunction) call.op,
+ call.operands,
+ w,
+ false,
+ false);
+ }
+ return super.visitCall(call);
+ }
+ });
+ }
+
+ /**
+ * Removes any casts that change nullability but not type.
+ *
+ * <p>For example, {@code CAST(1 = 0 AS BOOLEAN)} becomes {@code 1 = 0}.
+ */
+ public RexNode removeNullabilityCast(RexNode e) {
+ return RexUtil.removeNullabilityCast(rexBuilder.getTypeFactory(), e);
+ }
+
+ private static <C extends Comparable<C>> @Nullable RexNode processRange(
+ RexBuilder rexBuilder,
+ List<RexNode> terms,
+ Map<RexNode, Pair<Range<C>, List<RexNode>>> rangeTerms,
+ RexNode term,
+ RexNode ref,
+ C v0,
+ SqlKind comparison) {
+ Pair<Range<C>, List<RexNode>> p = rangeTerms.get(ref);
+ if (p == null) {
+ rangeTerms.put(ref, Pair.of(range(comparison, v0), ImmutableList.of(term)));
+ } else {
+ // Exists
+ boolean removeUpperBound = false;
+ boolean removeLowerBound = false;
+ Range<C> r = p.left;
+ final RexLiteral trueLiteral = rexBuilder.makeLiteral(true);
+ switch (comparison) {
+ case EQUALS:
+ if (!r.contains(v0)) {
+ // Range is empty, not satisfiable
+ return rexBuilder.makeLiteral(false);
+ }
+ rangeTerms.put(ref, Pair.of(Range.singleton(v0), ImmutableList.of(term)));
+ // remove
+ for (RexNode e : p.right) {
+ replaceLast(terms, e, trueLiteral);
+ }
+ break;
+ case LESS_THAN:
+ {
+ int comparisonResult = 0;
+ if (r.hasUpperBound()) {
+ comparisonResult = v0.compareTo(r.upperEndpoint());
+ }
+ if (comparisonResult <= 0) {
+ // 1) No upper bound, or
+ // 2) We need to open the upper bound, or
+ // 3) New upper bound is lower than old upper bound
+ if (r.hasLowerBound()) {
+ if (v0.compareTo(r.lowerEndpoint()) <= 0) {
+ // Range is empty, not satisfiable
+ return rexBuilder.makeLiteral(false);
+ }
+ // a <= x < b OR a < x < b
+ r =
+ Range.range(
+ r.lowerEndpoint(),
+ r.lowerBoundType(),
+ v0,
+ BoundType.OPEN);
+ } else {
+ // x < b
+ r = Range.lessThan(v0);
+ }
+
+ if (r.isEmpty()) {
+ // Range is empty, not satisfiable
+ return rexBuilder.makeLiteral(false);
+ }
+
+ // remove prev upper bound
+ removeUpperBound = true;
+ } else {
+ // Remove this term as it is contained in current upper bound
+ replaceLast(terms, term, trueLiteral);
+ }
+ break;
+ }
+ case LESS_THAN_OR_EQUAL:
+ {
+ int comparisonResult = -1;
+ if (r.hasUpperBound()) {
+ comparisonResult = v0.compareTo(r.upperEndpoint());
+ }
+ if (comparisonResult < 0) {
+ // 1) No upper bound, or
+ // 2) New upper bound is lower than old upper bound
+ if (r.hasLowerBound()) {
+ if (v0.compareTo(r.lowerEndpoint()) < 0) {
+ // Range is empty, not satisfiable
+ return rexBuilder.makeLiteral(false);
+ }
+ // a <= x <= b OR a < x <= b
+ r =
+ Range.range(
+ r.lowerEndpoint(),
+ r.lowerBoundType(),
+ v0,
+ BoundType.CLOSED);
+ } else {
+ // x <= b
+ r = Range.atMost(v0);
+ }
+
+ if (r.isEmpty()) {
+ // Range is empty, not satisfiable
+ return rexBuilder.makeLiteral(false);
+ }
+
+ // remove prev upper bound
+ removeUpperBound = true;
+ } else {
+ // Remove this term as it is contained in current upper bound
+ replaceLast(terms, term, trueLiteral);
+ }
+ break;
+ }
+ case GREATER_THAN:
+ {
+ int comparisonResult = 0;
+ if (r.hasLowerBound()) {
+ comparisonResult = v0.compareTo(r.lowerEndpoint());
+ }
+ if (comparisonResult >= 0) {
+ // 1) No lower bound, or
+ // 2) We need to open the lower bound, or
+ // 3) New lower bound is greater than old lower bound
+ if (r.hasUpperBound()) {
+ if (v0.compareTo(r.upperEndpoint()) >= 0) {
+ // Range is empty, not satisfiable
+ return rexBuilder.makeLiteral(false);
+ }
+ // a < x <= b OR a < x < b
+ r =
+ Range.range(
+ v0,
+ BoundType.OPEN,
+ r.upperEndpoint(),
+ r.upperBoundType());
+ } else {
+ // x > a
+ r = Range.greaterThan(v0);
+ }
+
+ if (r.isEmpty()) {
+ // Range is empty, not satisfiable
+ return rexBuilder.makeLiteral(false);
+ }
+
+ // remove prev lower bound
+ removeLowerBound = true;
+ } else {
+ // Remove this term as it is contained in current lower bound
+ replaceLast(terms, term, trueLiteral);
+ }
+ break;
+ }
+ case GREATER_THAN_OR_EQUAL:
+ {
+ int comparisonResult = 1;
+ if (r.hasLowerBound()) {
+ comparisonResult = v0.compareTo(r.lowerEndpoint());
+ }
+ if (comparisonResult > 0) {
+ // 1) No lower bound, or
+ // 2) New lower bound is greater than old lower bound
+ if (r.hasUpperBound()) {
+ if (v0.compareTo(r.upperEndpoint()) > 0) {
+ // Range is empty, not satisfiable
+ return rexBuilder.makeLiteral(false);
+ }
+ // a <= x <= b OR a <= x < b
+ r =
+ Range.range(
+ v0,
+ BoundType.CLOSED,
+ r.upperEndpoint(),
+ r.upperBoundType());
+ } else {
+ // x >= a
+ r = Range.atLeast(v0);
+ }
+
+ if (r.isEmpty()) {
+ // Range is empty, not satisfiable
+ return rexBuilder.makeLiteral(false);
+ }
+
+ // remove prev lower bound
+ removeLowerBound = true;
+ } else {
+ // Remove this term as it is contained in current lower bound
+ replaceLast(terms, term, trueLiteral);
+ }
+ break;
+ }
+ default:
+ throw new AssertionError();
+ }
+ if (removeUpperBound) {
+ ImmutableList.Builder<RexNode> newBounds = ImmutableList.builder();
+ for (RexNode e : p.right) {
+ if (isUpperBound(e)) {
+ replaceLast(terms, e, trueLiteral);
+ } else {
+ newBounds.add(e);
+ }
+ }
+ newBounds.add(term);
+ rangeTerms.put(ref, Pair.of(r, newBounds.build()));
+ } else if (removeLowerBound) {
+ ImmutableList.Builder<RexNode> newBounds = ImmutableList.builder();
+ for (RexNode e : p.right) {
+ if (isLowerBound(e)) {
+ replaceLast(terms, e, trueLiteral);
+ } else {
+ newBounds.add(e);
+ }
+ }
+ newBounds.add(term);
+ rangeTerms.put(ref, Pair.of(r, newBounds.build()));
+ }
+ }
+ // Default
+ return null;
+ }
+
+ private static <C extends Comparable<C>> Range<C> range(SqlKind comparison, C c) {
+ switch (comparison) {
+ case EQUALS:
+ return Range.singleton(c);
+ case LESS_THAN:
+ return Range.lessThan(c);
+ case LESS_THAN_OR_EQUAL:
+ return Range.atMost(c);
+ case GREATER_THAN:
+ return Range.greaterThan(c);
+ case GREATER_THAN_OR_EQUAL:
+ return Range.atLeast(c);
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ private static <C extends Comparable<C>> RangeSet<C> rangeSet(SqlKind comparison, C c) {
+ switch (comparison) {
+ case EQUALS:
+ case LESS_THAN:
+ case LESS_THAN_OR_EQUAL:
+ case GREATER_THAN:
+ case GREATER_THAN_OR_EQUAL:
+ return ImmutableRangeSet.of(range(comparison, c));
+ case NOT_EQUALS:
+ return ImmutableRangeSet.<C>builder()
+ .add(range(SqlKind.LESS_THAN, c))
+ .add(range(SqlKind.GREATER_THAN, c))
+ .build();
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ /** Marker interface for predicates (expressions that evaluate to BOOLEAN). */
+ private interface Predicate {
+ /** Wraps an expression in a Predicate or returns null. */
+ static @Nullable Predicate of(RexNode t) {
+ final Predicate p = Comparison.of(t);
+ if (p != null) {
+ return p;
+ }
+ return IsPredicate.of(t);
+ }
+
+ /** Returns whether this predicate can be used while simplifying other OR operands. */
+ default boolean allowedInOr(RelOptPredicateList predicates) {
+ return true;
+ }
+ }
+
+ /** Visitor which finds all inputs used by an expressions. */
+ private static class VariableCollector extends RexVisitorImpl<Void> {
+ private final Set<RexInputRef> refs = new HashSet<>();
+
+ VariableCollector() {
+ super(true);
+ }
+
+ @Override
+ public Void visitInputRef(RexInputRef inputRef) {
+ refs.add(inputRef);
+ return super.visitInputRef(inputRef);
+ }
+ }
+
+ /**
+ * Represents a simple Comparison.
+ *
+ * <p>Left hand side is a {@link RexNode}, right hand side is a literal.
+ */
+ private static class Comparison implements Predicate {
+ final RexNode ref;
+ final SqlKind kind;
+ final RexLiteral literal;
+
+ private Comparison(RexNode ref, SqlKind kind, RexLiteral literal) {
+ this.ref = requireNonNull(ref, "ref");
+ this.kind = requireNonNull(kind, "kind");
+ this.literal = requireNonNull(literal, "literal");
+ }
+
+ /**
+ * Creates a comparison, between a {@link RexInputRef} or {@link RexFieldAccess} or
+ * deterministic {@link RexCall} and a literal.
+ */
+ static @Nullable Comparison of(RexNode e) {
+ return of(
+ e,
+ node ->
+ RexUtil.isReferenceOrAccess(node, true)
+ || RexUtil.isDeterministic(node));
+ }
+
+ /** Creates a comparison, or returns null. */
+ static @Nullable Comparison of(
+ RexNode e, java.util.function.Predicate<RexNode> nodePredicate) {
+ switch (e.getKind()) {
+ case EQUALS:
+ case NOT_EQUALS:
+ case LESS_THAN:
+ case GREATER_THAN:
+ case LESS_THAN_OR_EQUAL:
+ case GREATER_THAN_OR_EQUAL:
+ final RexCall call = (RexCall) e;
+ final RexNode left = call.getOperands().get(0);
+ final RexNode right = call.getOperands().get(1);
+ switch (right.getKind()) {
+ case LITERAL:
+ if (nodePredicate.test(left)) {
+ return new Comparison(left, e.getKind(), (RexLiteral) right);
+ }
+ break;
+ default:
+ break;
+ }
+ switch (left.getKind()) {
+ case LITERAL:
+ if (nodePredicate.test(right)) {
+ return new Comparison(
+ right, e.getKind().reverse(), (RexLiteral) left);
+ }
+ break;
+ default:
+ break;
+ }
+ break;
+ default:
+ break;
+ }
+ return null;
+ }
+
+ @Override
+ public boolean allowedInOr(RelOptPredicateList predicates) {
+ // if ref is not a 'loss-less' cast then can't be allowed to be used
+ // while simplifying other OR operands
+ if (ref.isA(SqlKind.CAST) && !RexUtil.isLosslessCast(ref)) {
+ return false;
+ }
+ return !ref.getType().isNullable() || predicates.isEffectivelyNotNull(ref);
+ }
+ }
+
+ /** Represents an IS Predicate. */
+ private static class IsPredicate implements Predicate {
+ final RexNode ref;
+ final SqlKind kind;
+
+ private IsPredicate(RexNode ref, SqlKind kind) {
+ this.ref = requireNonNull(ref, "ref");
+ this.kind = requireNonNull(kind, "kind");
+ }
+
+ /** Creates an IS predicate, or returns null. */
+ static @Nullable IsPredicate of(RexNode e) {
+ switch (e.getKind()) {
+ case IS_NULL:
+ case IS_NOT_NULL:
+ RexNode pA = ((RexCall) e).getOperands().get(0);
+ if (!(RexUtil.isReferenceOrAccess(pA, true) || RexUtil.isDeterministic(pA))) {
+ return null;
+ }
+ return new IsPredicate(pA, e.getKind());
+ default:
+ break;
+ }
+ return null;
+ }
+ }
+
+ private static boolean isUpperBound(final RexNode e) {
+ final List<RexNode> operands;
+ switch (e.getKind()) {
+ case LESS_THAN:
+ case LESS_THAN_OR_EQUAL:
+ operands = ((RexCall) e).getOperands();
+ return RexUtil.isReferenceOrAccess(operands.get(0), true)
+ && operands.get(1).isA(SqlKind.LITERAL);
+ case GREATER_THAN:
+ case GREATER_THAN_OR_EQUAL:
+ operands = ((RexCall) e).getOperands();
+ return RexUtil.isReferenceOrAccess(operands.get(1), true)
+ && operands.get(0).isA(SqlKind.LITERAL);
+ default:
+ return false;
+ }
+ }
+
+ private static boolean isLowerBound(final RexNode e) {
+ final List<RexNode> operands;
+ switch (e.getKind()) {
+ case LESS_THAN:
+ case LESS_THAN_OR_EQUAL:
+ operands = ((RexCall) e).getOperands();
+ return RexUtil.isReferenceOrAccess(operands.get(1), true)
+ && operands.get(0).isA(SqlKind.LITERAL);
+ case GREATER_THAN:
+ case GREATER_THAN_OR_EQUAL:
+ operands = ((RexCall) e).getOperands();
+ return RexUtil.isReferenceOrAccess(operands.get(0), true)
+ && operands.get(1).isA(SqlKind.LITERAL);
+ default:
+ return false;
+ }
+ }
+
+ /**
+ * Combines predicates AND, optimizes, and returns null if the result is always false.
+ *
+ * <p>The expression is simplified on the assumption that an UNKNOWN value is always treated as
+ * FALSE. Therefore the simplified expression may sometimes evaluate to FALSE where the original
+ * expression would evaluate to UNKNOWN.
+ *
+ * @param predicates Filter condition predicates
+ * @return simplified conjunction of predicates for the filter, null if always false
+ */
+ public @Nullable RexNode simplifyFilterPredicates(Iterable<? extends RexNode> predicates) {
+ final RexNode simplifiedAnds =
+ withPredicateElimination(Bug.CALCITE_2401_FIXED)
+ .simplifyUnknownAsFalse(RexUtil.composeConjunction(rexBuilder, predicates));
+ if (simplifiedAnds.isAlwaysFalse()) {
+ return null;
+ }
+
+ // Remove cast of BOOLEAN NOT NULL to BOOLEAN or vice versa. Filter accepts
+ // nullable and not-nullable conditions, but a CAST might get in the way of
+ // other rewrites.
+ return removeNullabilityCast(simplifiedAnds);
+ }
+
+ /**
+ * Replaces the last occurrence of one specified value in a list with another.
+ *
+ * <p>Does not change the size of the list.
+ *
+ * <p>Returns whether the value was found.
+ */
+ private static <E> boolean replaceLast(List<E> list, E oldVal, E newVal) {
+ @SuppressWarnings("argument.type.incompatible")
+ final int index = list.lastIndexOf(oldVal);
+ if (index < 0) {
+ return false;
+ }
+ list.set(index, newVal);
+ return true;
+ }
+
+ /** Gathers expressions that can be converted into {@link Sarg search arguments}. */
+ static class SargCollector {
+ final Map<RexNode, RexSargBuilder> map = new HashMap<>();
+ private final RexBuilder rexBuilder;
+ private final boolean negate;
+
+ /**
+ * Count of the new terms after converting all the operands to {@code SEARCH} on a {@link
+ * Sarg}. It is used to decide whether the new terms are simpler.
+ */
+ private int newTermsCount;
+
+ SargCollector(RexBuilder rexBuilder, boolean negate) {
+ this.rexBuilder = rexBuilder;
+ this.negate = negate;
+ }
+
+ /**
+ * Accepts an expression and converts it to a Sarg if possible.
+ *
+ * @param term the expression to convert
+ * @param newTerms the list holding the result of the conversion or the original term if it
+ * cannot be converted
+ */
+ private void accept(RexNode term, List<RexNode> newTerms) {
+ if (!accept_(term, newTerms)) {
+ newTerms.add(term);
+ }
+ newTermsCount = newTerms.size();
+ }
+
+ /**
+ * Accepts an expression and converts it to a Sarg if possible. Only certain kinds of
+ * expressions can be converted to Sargs.
+ *
+ * @param e the expression to convert
+ * @param newTerms the list to which the Sarg will be added if the expression is accepted
+ * @return true if the expression can be converted to a Sarg, false otherwise
+ */
+ private boolean accept_(RexNode e, List<RexNode> newTerms) {
+ switch (e.getKind()) {
+ case LESS_THAN:
+ case LESS_THAN_OR_EQUAL:
+ case GREATER_THAN:
+ case GREATER_THAN_OR_EQUAL:
+ case EQUALS:
+ case NOT_EQUALS:
+ case SEARCH:
+ case IS_NOT_DISTINCT_FROM:
+ case IS_DISTINCT_FROM:
+ return accept2(
+ ((RexCall) e).operands.get(0),
+ ((RexCall) e).operands.get(1),
+ e.getKind(),
+ newTerms);
+ case IS_NULL:
+ case IS_NOT_NULL:
+ final RexNode arg = ((RexCall) e).operands.get(0);
+ return accept1(arg, e.getKind(), newTerms);
+ default:
+ return false;
+ }
+ }
+
+ /**
+ * Accepts two operands from a binary comparison operator and converts them to a Sarg if
+ * possible. Only comparisons between a literal and a deterministic expression can be
+ * converted. Simplifications with non-deterministic expressions are generally avoided to
+ * ensure consistent results.
+ *
+ * @param left the left operand of the comparison
+ * @param right the right operand of the comparison
+ * @param kind the kind of comparison operator
+ * @param newTerms the list to which the Sarg will be added if accepted
+ * @return true if the operands can be converted to a Sarg, false otherwise
+ */
+ private boolean accept2(RexNode left, RexNode right, SqlKind kind, List<RexNode> newTerms) {
+ if (right.isA(SqlKind.LITERAL) && RexUtil.isDeterministic(left)) {
+ return accept2b(left, kind, (RexLiteral) right, newTerms);
+ }
+ if (left.isA(SqlKind.LITERAL) && RexUtil.isDeterministic(right)) {
+ return accept2b(right, kind.reverse(), (RexLiteral) left, newTerms);
+ }
+ return false;
+ }
+
+ private static <E> E addFluent(List<? super E> list, E e) {
+ list.add(e);
+ return e;
+ }
+
+ /**
+ * Accepts an operand from a null predicate and converts it to a Sarg.
+ *
+ * @param e the operand of the null predicate
+ * @param kind the kind of the null predicate
+ * @param newTerms the list to which the Sarg is added
+ * @return true since the operand is always converted to a Sarg
+ */
+ private boolean accept1(RexNode e, SqlKind kind, List<RexNode> newTerms) {
+ final RexSargBuilder b =
+ map.computeIfAbsent(
+ e,
+ e2 -> addFluent(newTerms, new RexSargBuilder(e2, rexBuilder, negate)));
+ switch (negate ? kind.negate() : kind) {
+ case IS_NULL:
+ b.nullAs = b.nullAs.or(TRUE);
+ return true;
+ case IS_NOT_NULL:
+ b.nullAs = b.nullAs.or(FALSE);
+ b.addAll();
+ return true;
+ default:
+ throw new AssertionError("unexpected " + kind);
+ }
+ }
+
+ /**
+ * Accepts two operands from a binary comparison operator and converts them to a Sarg when
+ * the literal is not null. The conversion depends on the kind of comparison operator and
+ * only certain operators are supported.
+ *
+ * @param e any kind of deterministic expression
+ * @param kind the kind of comparison operator
+ * @param literal the literal operand of the comparison
+ * @param newTerms the list to which the Sarg is added if accepted
+ * @return false if the literal operand is null, true otherwise
+ */
+ private boolean accept2b(
+ RexNode e, SqlKind kind, RexLiteral literal, List<RexNode> newTerms) {
+ if (literal.getValue() == null) {
+ // Cannot include expressions 'x > NULL' in a Sarg. Comparing to a NULL
+ // literal is a daft thing to do, because it always returns UNKNOWN. It
+ // is better handled by other simplifications.
+ return false;
+ }
+ final RexSargBuilder b =
+ map.computeIfAbsent(
+ e,
+ e2 -> addFluent(newTerms, new RexSargBuilder(e2, rexBuilder, negate)));
+ if (negate) {
+ kind = kind.negateNullSafe();
+ }
+ final Comparable value = requireNonNull(literal.getValueAs(Comparable.class), "value");
+ switch (kind) {
+ case LESS_THAN:
+ b.addRange(Range.lessThan(value), literal.getType());
+ return true;
+ case LESS_THAN_OR_EQUAL:
+ b.addRange(Range.atMost(value), literal.getType());
+ return true;
+ case GREATER_THAN:
+ b.addRange(Range.greaterThan(value), literal.getType());
+ return true;
+ case GREATER_THAN_OR_EQUAL:
+ b.addRange(Range.atLeast(value), literal.getType());
+ return true;
+ case EQUALS:
+ b.addRange(Range.singleton(value), literal.getType());
+ return true;
+ case IS_NOT_DISTINCT_FROM:
+ b.addRange(Range.singleton(value), literal.getType(), FALSE);
+ return true;
+ case NOT_EQUALS:
+ b.addRange(Range.lessThan(value), literal.getType());
+ b.addRange(Range.greaterThan(value), literal.getType());
+ return true;
+ case IS_DISTINCT_FROM:
+ b.addRange(Range.lessThan(value), literal.getType(), TRUE);
+ b.addRange(Range.greaterThan(value), literal.getType(), TRUE);
+ return true;
+ case SEARCH:
+ final Sarg sarg = (Sarg) value;
+ b.addSarg(sarg, negate, literal.getType());
+ return true;
+ default:
+ throw new AssertionError("unexpected " + kind);
+ }
+ }
+
+ /** Returns whether it is worth to fix and convert to {@code SEARCH} calls. */
+ boolean needToFix() {
+ // Fix and converts to SEARCH if:
+ // 1. A Sarg has complexity greater than 1;
+ // 2. A Sarg was merged with another Sarg or range;
+ // 3. The terms are reduced as simpler Sarg points;
+ // 4. The terms are reduced as simpler Sarg comparison.
+
+ // Ignore 'negate' just to be compatible with previous versions of this
+ // method. "build().complexity()" would be a better estimate, if we could
+ // switch to it breaking lots of plans.
+ final Collection<RexSargBuilder> builders = map.values();
+ return builders.stream().anyMatch(b -> b.build(false).complexity() > 1 || b.mergedSarg)
+ || newTermsCount == 1 && builders.stream().allMatch(b -> simpleSarg(b.build()));
+ }
+
+ /**
+ * Returns whether this Sarg can be expanded to more simple form, e.g. the IN call or single
+ * comparison.
+ */
+ private static boolean simpleSarg(Sarg sarg) {
+ return sarg.isPoints()
+ || RangeSets.isOpenInterval(sarg.rangeSet)
+ || sarg.isComplementedPoints();
+ }
+
+ /**
+ * If a term is a call to {@code SEARCH} on a {@link RexSargBuilder}, converts it to a
+ * {@code SEARCH} on a {@link Sarg}.
+ */
+ static RexNode fix(RexBuilder rexBuilder, RexNode term, RexUnknownAs unknownAs) {
+ if (term instanceof RexSargBuilder) {
+ final RexSargBuilder sargBuilder = (RexSargBuilder) term;
+ final Sarg sarg = sargBuilder.build();
+ boolean isSmall = sarg.complexity() <= 1 || sarg.isAll() || sarg.isNone();
+ if (isSmall && simpleSarg(sarg)) {
+ // Expand small sargs into comparisons in order to avoid plan changes
+ // and better readability.
+ return RexUtil.sargRef(
+ rexBuilder, sargBuilder.ref, sarg, term.getType(), unknownAs);
+ }
+ return rexBuilder.makeCall(
+ SqlStdOperatorTable.SEARCH,
+ sargBuilder.ref,
+ rexBuilder.makeSearchArgumentLiteral(sarg, term.getType()));
+ }
+ return term;
+ }
+ }
+
+ /**
+ * Equivalent to a {@link RexLiteral} whose value is a {@link Sarg}, but mutable, so that the
+ * Sarg can be expanded as {@link SargCollector} traverses a list of OR or AND terms.
+ *
+ * <p>The {@link SargCollector#fix} method converts it to an immutable literal.
+ *
+ * <p>The {@link #nullAs} field will become {@link Sarg#nullAs}, as follows:
+ *
+ * <ul>
+ * <li>If there is at least one term that returns TRUE when the argument is NULL, then the
+ * overall value will be TRUE; failing that,
+ * <li>if there is at least one term that returns UNKNOWN when the argument is NULL, then the
+ * overall value will be UNKNOWN; failing that,
+ * <li>the value will be FALSE.
+ * </ul>
+ *
+ * <p>This is analogous to the behavior of OR in three-valued logic: {@code TRUE OR UNKNOWN OR
+ * FALSE} returns {@code TRUE}; {@code UNKNOWN OR FALSE OR UNKNOWN} returns {@code UNKNOWN};
+ * {@code FALSE OR FALSE} returns {@code FALSE}.
+ */
+ private static class RexSargBuilder extends RexNode {
+ final RexNode ref;
+ final RexBuilder rexBuilder;
+ final boolean negate;
+ final List<RelDataType> types = new ArrayList<>();
+ final RangeSet<Comparable> rangeSet = TreeRangeSet.create();
+ boolean hasSarg;
+ boolean mergedSarg;
+ RexUnknownAs nullAs = FALSE;
+
+ RexSargBuilder(RexNode ref, RexBuilder rexBuilder, boolean negate) {
+ this.ref = requireNonNull(ref, "ref");
+ this.rexBuilder = requireNonNull(rexBuilder, "rexBuilder");
+ this.negate = negate;
+ }
+
+ @Override
+ public String toString() {
+ return "SEARCH("
+ + ref
+ + ", "
+ + (negate ? "NOT " : "")
+ + rangeSet
+ + "; NULL AS "
+ + nullAs
+ + ")";
+ }
+
+ <C extends Comparable<C>> Sarg<C> build() {
+ return build(negate);
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ <C extends Comparable<C>> Sarg<C> build(boolean negate) {
+ final RangeSet<C> r = (RangeSet) this.rangeSet;
+ if (negate) {
+ return Sarg.of(nullAs.negate(), r.complement());
+ } else {
+ return Sarg.of(nullAs, r);
+ }
+ }
+
+ @Override
+ public RelDataType getType() {
+ if (this.types.isEmpty()) {
+ // Expression is "x IS NULL"
+ return ref.getType();
+ }
+ final List<RelDataType> distinctTypes = Util.distinctList(this.types);
+ return requireNonNull(
+ rexBuilder.typeFactory.leastRestrictive(distinctTypes),
+ () -> "Can't find leastRestrictive type among " + distinctTypes);
+ }
+
+ @Override
+ public <R> R accept(RexVisitor<R> visitor) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <R, P> R accept(RexBiVisitor<R, P> visitor, P arg) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean equals(@Nullable Object obj) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int hashCode() {
+ throw new UnsupportedOperationException();
+ }
+
+ void addAll() {
+ rangeSet.add(Range.all());
+ }
+
+ void addRange(Range<Comparable> range, RelDataType type) {
+ addRange(range, type, UNKNOWN);
+ }
+
+ void addRange(Range<Comparable> range, RelDataType type, RexUnknownAs unknownAs) {
+ types.add(type);
+ rangeSet.add(range);
+ mergedSarg |= hasSarg;
+ nullAs = nullAs.or(unknownAs);
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ void addSarg(Sarg sarg, boolean negate, RelDataType type) {
+ final RangeSet r;
+ final RexUnknownAs nullAs;
+ if (negate) {
+ r = sarg.rangeSet.complement();
+ nullAs = sarg.nullAs.negate();
+ } else {
+ r = sarg.rangeSet;
+ nullAs = sarg.nullAs;
+ }
+ types.add(type);
+ rangeSet.addAll(r);
+ mergedSarg |= !rangeSet.isEmpty();
+ hasSarg = true;
+ switch (nullAs) {
+ case TRUE:
+ this.nullAs = this.nullAs.or(TRUE);
+ break;
+ case FALSE:
+ this.nullAs = this.nullAs.or(FALSE);
+ break;
+ case UNKNOWN:
+ this.nullAs = this.nullAs.or(UNKNOWN);
+ break;
+ }
+ }
+ }
+}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rex/RexUtil.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rex/RexUtil.java
index 7449e9162ff..a14afe0176d 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rex/RexUtil.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/rex/RexUtil.java
@@ -25,12 +25,15 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Range;
import org.apache.calcite.DataContexts;
import org.apache.calcite.linq4j.function.Predicate1;
+import org.apache.calcite.plan.PlanTooComplexError;
import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelCollations;
import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Calc;
+import org.apache.calcite.rel.core.CorrelationId;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.Project;
@@ -804,6 +807,11 @@ public class RexUtil {
public Boolean visitLambdaRef(RexLambdaRef lambdaRef) {
return false;
}
+
+ @Override
+ public Boolean visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex) {
+ return false;
+ }
}
/**
@@ -843,13 +851,13 @@ public class RexUtil {
}
public static List<RexNode> retainDeterministic(List<RexNode> list) {
- List<RexNode> conjuctions = new ArrayList<>();
+ List<RexNode> conjunctions = new ArrayList<>();
for (RexNode x : list) {
if (isDeterministic(x)) {
- conjuctions.add(x);
+ conjunctions.add(x);
}
}
- return conjuctions;
+ return conjunctions;
}
/**
@@ -1658,22 +1666,31 @@ public class RexUtil {
* @return 'true' when the conversion can certainly be determined to be loss-less cast, but may
* return 'false' for some lossless casts.
*/
- @API(since = "1.22", status = API.Status.EXPERIMENTAL)
+ @API(since = "1.22", status = API.Status.STABLE)
public static boolean isLosslessCast(RelDataType source, RelDataType target) {
final SqlTypeName sourceSqlTypeName = source.getSqlTypeName();
final SqlTypeName targetSqlTypeName = target.getSqlTypeName();
- // 1) Both INT numeric types
- if (SqlTypeFamily.INTEGER.getTypeNames().contains(sourceSqlTypeName)
- && SqlTypeFamily.INTEGER.getTypeNames().contains(targetSqlTypeName)) {
- return targetSqlTypeName.compareTo(sourceSqlTypeName) >= 0;
+
+ // INT -> INT: use range containment (signed/unsigned)
+ if (SqlTypeUtil.isIntType(source) && SqlTypeUtil.isIntType(target)) {
+ final boolean sourceIsUnsigned =
+ SqlTypeFamily.UNSIGNED_NUMERIC.getTypeNames().contains(sourceSqlTypeName);
+ final boolean targetIsUnsigned =
+ SqlTypeFamily.UNSIGNED_NUMERIC.getTypeNames().contains(targetSqlTypeName);
+ if (!sourceIsUnsigned && targetIsUnsigned) {
+ return false;
+ }
+ return SqlTypeUtil.integerRangeContains(target, source);
}
- // 2) Both CHARACTER types: it depends on the precision (length)
+
+ // CHARACTER -> CHARACTER: valid if target order >= source and length grows
if (SqlTypeFamily.CHARACTER.getTypeNames().contains(sourceSqlTypeName)
&& SqlTypeFamily.CHARACTER.getTypeNames().contains(targetSqlTypeName)) {
return targetSqlTypeName.compareTo(sourceSqlTypeName) >= 0
&& source.getPrecision() <= target.getPrecision();
}
- // 3) From NUMERIC family to CHARACTER family: it depends on the precision/scale
+
+ // NUMERIC -> CHARACTER: allow when target length accommodates sign/scale
if (sourceSqlTypeName.getFamily() == SqlTypeFamily.NUMERIC
&& targetSqlTypeName.getFamily() == SqlTypeFamily.CHARACTER) {
int sourceLength = source.getPrecision() + 1; // include sign
@@ -1683,7 +1700,66 @@ public class RexUtil {
final int targetPrecision = target.getPrecision();
return targetPrecision == PRECISION_NOT_SPECIFIED || targetPrecision >= sourceLength;
}
- // Return FALSE by default
+
+ // DECIMAL -> DECIMAL: allow when precision/scale can only expand
+ if (sourceSqlTypeName == SqlTypeName.DECIMAL && targetSqlTypeName == SqlTypeName.DECIMAL) {
+ int sourcePrecision = source.getPrecision();
+ int sourceScale = Math.max(source.getScale(), 0);
+ int targetPrecision = target.getPrecision();
+ int targetScale = Math.max(target.getScale(), 0);
+ if (sourcePrecision <= 0 || targetPrecision <= 0) {
+ return false;
+ }
+ return targetScale >= sourceScale
+ && (targetPrecision - targetScale) >= (sourcePrecision - sourceScale);
+ }
+
+ // INT (signed/unsigned) -> DECIMAL: valid if integer digits fit within target precision
+ if (SqlTypeUtil.isIntType(source) && targetSqlTypeName == SqlTypeName.DECIMAL) {
+ int targetPrecision = target.getPrecision();
+ int targetScale = Math.max(target.getScale(), 0);
+ int sourcePrecision = source.getPrecision();
+ return sourcePrecision > 0 && (targetPrecision - targetScale) >= sourcePrecision;
+ }
+
+ // DECIMAL -> INT: only when scale = 0 and range fits target integer type
+ if (sourceSqlTypeName == SqlTypeName.DECIMAL && SqlTypeUtil.isIntType(target)) {
+ if (source.getScale() != 0) {
+ return false;
+ }
+ return SqlTypeUtil.integerRangeContains(target, source);
+ }
+
+ // APPROXIMATE NUMERIC -> APPROXIMATE NUMERIC: allow when target precision >= source
+ // precision
+ if (SqlTypeFamily.APPROXIMATE_NUMERIC.getTypeNames().contains(sourceSqlTypeName)
+ && SqlTypeFamily.APPROXIMATE_NUMERIC.getTypeNames().contains(targetSqlTypeName)) {
+ // Lossless if target has at least as many significant digits as source
+ final int sourcePrecision = source.getPrecision();
+ final int targetPrecision = target.getPrecision();
+ return targetPrecision >= sourcePrecision;
+ }
+
+ // EXACT NUMERIC -> APPROXIMATE NUMERIC: allow only for scale=0 values within target digits
+ if (SqlTypeFamily.EXACT_NUMERIC.getTypeNames().contains(sourceSqlTypeName)
+ && SqlTypeFamily.APPROXIMATE_NUMERIC.getTypeNames().contains(targetSqlTypeName)) {
+ final int targetPrecision = target.getPrecision();
+
+ // DECIMAL -> APPROXIMATE NUMERIC
+ if (sourceSqlTypeName == SqlTypeName.DECIMAL) {
+ final int sourcePrecision = source.getPrecision();
+ if (sourcePrecision <= 0 || source.getScale() != 0) {
+ return false;
+ }
+ // scale is 0, just check precision
+ return sourcePrecision <= targetPrecision;
+ }
+
+ // INT (signed/unsigned) -> APPROXIMATE NUMERIC
+ int sourcePrecision = source.getPrecision();
+ return sourcePrecision > 0 && sourcePrecision <= targetPrecision;
+ }
+
return false;
}
@@ -1864,6 +1940,36 @@ public class RexUtil {
});
}
+ /**
+ * Shifts every {@link RexFieldAccess} with {@link CorrelationId} in an {@link RelNode} by
+ * {@code offset}.
+ */
+ public static RelNode shiftFieldAccess(
+ RexBuilder rexBuilder,
+ RelNode node,
+ final CorrelationId id,
+ RelNode outer,
+ final int offset) {
+ if (offset == 0) {
+ return node;
+ }
+
+ RexNode correl = rexBuilder.makeCorrel(outer.getRowType(), id);
+ return node.accept(
+ new RexShuttle() {
+ @Override
+ public RexNode visitFieldAccess(RexFieldAccess fieldAccess) {
+ if (fieldAccess.getReferenceExpr() instanceof RexCorrelVariable
+ && ((RexCorrelVariable) fieldAccess.getReferenceExpr())
+ .id.equals(id)) {
+ return rexBuilder.makeFieldAccess(
+ correl, fieldAccess.getField().getIndex() + offset);
+ }
+ return fieldAccess;
+ }
+ });
+ }
+
/**
* Creates an equivalent version of a node where common factors among ORs are pulled up.
*
@@ -2983,12 +3089,6 @@ public class RexUtil {
}
}
- /**
- * Exception to catch when optimizing the plan produces a result that is too complex, either at
- * the Rel or at the Rex level.
- */
- private static class PlanTooComplexError extends ControlFlowException {}
-
/**
* Visitor that throws {@link org.apache.calcite.util.Util.FoundOne} if applied to an expression
* that contains a {@link RexCorrelVariable}.
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
index e5a41b36323..781331b9b81 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
@@ -42,6 +42,7 @@ import org.apache.calcite.linq4j.function.Functions;
import org.apache.calcite.linq4j.function.NonDeterministic;
import org.apache.calcite.linq4j.function.Predicate1;
import org.apache.calcite.linq4j.tree.Primitive;
+import org.apache.calcite.linq4j.tree.UnsignedType;
import org.apache.calcite.rel.type.TimeFrame;
import org.apache.calcite.rel.type.TimeFrameSet;
import org.apache.calcite.runtime.FlatLists.ComparableList;
@@ -65,10 +66,15 @@ import org.apache.commons.codec.binary.Base32;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.codec.language.Soundex;
-import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.math3.util.CombinatoricsUtils;
+import org.apache.commons.text.StringEscapeUtils;
import org.apache.commons.text.similarity.LevenshteinDistance;
import org.checkerframework.checker.nullness.qual.Nullable;
+import org.joou.UByte;
+import org.joou.UInteger;
+import org.joou.ULong;
+import org.joou.UShort;
+import org.joou.Unsigned;
import java.lang.reflect.Field;
import java.math.BigDecimal;
@@ -2530,6 +2536,22 @@ public class SqlFunctions {
throw notArithmetic("+", b0, b1);
}
+ public static UByte plus(UByte b0, UByte b1) {
+ return (b0 == null || b1 == null) ? castNonNull(null) : b0.add(b1);
+ }
+
+ public static UShort plus(UShort b0, UShort b1) {
+ return (b0 == null || b1 == null) ? castNonNull(null) : b0.add(b1);
+ }
+
+ public static UInteger plus(UInteger b0, UInteger b1) {
+ return (b0 == null || b1 == null) ? castNonNull(null) : b0.add(b1);
+ }
+
+ public static ULong plus(ULong b0, ULong b1) {
+ return (b0 == null || b1 == null) ? castNonNull(null) : b0.add(b1);
+ }
+
// checked +
static byte intToByte(int value) {
@@ -2564,6 +2586,22 @@ public class SqlFunctions {
return Math.addExact(b0, b1);
}
+ public static UByte checkedPlus(UByte b0, UByte b1) {
+ return b0.add(b1);
+ }
+
+ public static UShort checkedPlus(UShort b0, UShort b1) {
+ return b0.add(b1);
+ }
+
+ public static UInteger checkedPlus(UInteger b0, UInteger b1) {
+ return b0.add(b1);
+ }
+
+ public static ULong checkedPlus(ULong b0, ULong b1) {
+ return b0.add(b1);
+ }
+
// -
/** SQL <code>-</code> operator applied to int values. */
@@ -2596,6 +2634,11 @@ public class SqlFunctions {
return (b0 == null || b1 == null) ? castNonNull(null) : (b0.longValue() - b1.longValue());
}
+ /** SQL <code>-</code> operator applied to nullable long and long values. */
+ public static Long minus(Long b0, Long b1) {
+ return (b0 == null || b1 == null) ? castNonNull(null) : b0.longValue() - b1.longValue();
+ }
+
/** SQL <code>-</code> operator applied to nullable BigDecimal values. */
public static BigDecimal minus(BigDecimal b0, BigDecimal b1) {
return (b0 == null || b1 == null) ? castNonNull(null) : b0.subtract(b1);
@@ -2617,6 +2660,23 @@ public class SqlFunctions {
throw notArithmetic("-", b0, b1);
}
+ public static UByte minus(UByte b0, UByte b1) {
+ return (b0 == null || b1 == null) ? castNonNull(null) : b0.subtract(b1);
+ }
+
+ public static UShort minus(UShort b0, UShort b1) {
+ return (b0 == null || b1 == null) ? castNonNull(null) : b0.subtract(b1);
+ }
+
+ public static UInteger minus(UInteger b0, UInteger b1) {
+ return (b0 == null || b1 == null) ? castNonNull(null) : b0.subtract(b1);
+ }
+
+ /** SQL <code>-</code> operator applied to nullable unsigned long and long values. */
+ public static ULong minus(ULong b0, ULong b1) {
+ return (b0 == null || b1 == null) ? castNonNull(null) : b0.subtract(b1);
+ }
+
// checked -
public static byte checkedMinus(byte b0, byte b1) {
@@ -2651,6 +2711,38 @@ public class SqlFunctions {
return Math.subtractExact(0, b);
}
+ public static UByte checkedMinus(UByte b0, UByte b1) {
+ return b0.subtract(b1);
+ }
+
+ public static UShort checkedMinus(UShort b0, UShort b1) {
+ return b0.subtract(b1);
+ }
+
+ public static UInteger checkedMinus(UInteger b0, UInteger b1) {
+ return b0.subtract(b1);
+ }
+
+ public static ULong checkedMinus(ULong b0, ULong b1) {
+ return b0.subtract(b1);
+ }
+
+ public static UByte checkedUnaryMinus(UByte b) {
+ return Unsigned.ubyte(0).subtract(b);
+ }
+
+ public static UShort checkedUnaryMinus(UShort b) {
+ return Unsigned.ushort(0).subtract(b);
+ }
+
+ public static UInteger checkedUnaryMinus(UInteger b) {
+ return Unsigned.uint(0).subtract(b);
+ }
+
+ public static ULong checkedUnaryMinus(ULong b) {
+ return Unsigned.ulong(0).subtract(b);
+ }
+
// /
/** SQL <code>/</code> operator applied to int values. */
@@ -2714,6 +2806,31 @@ public class SqlFunctions {
return BigDecimal.valueOf(b0).divide(b1, RoundingMode.HALF_DOWN).longValue();
}
+ public static UByte divide(UByte b0, UByte b1) {
+ return (b0 == null || b1 == null)
+ ? castNonNull(null)
+ : UByte.valueOf(b0.intValue() / b1.intValue());
+ }
+
+ public static UShort divide(UShort b0, UShort b1) {
+ return (b0 == null || b1 == null)
+ ? castNonNull(null)
+ : UShort.valueOf(b0.intValue() / b1.intValue());
+ }
+
+ public static UInteger divide(UInteger b0, UInteger b1) {
+ return (b0 == null || b1 == null)
+ ? castNonNull(null)
+ : UInteger.valueOf(b0.longValue() / b1.longValue());
+ }
+
+ public static ULong divide(ULong b0, ULong b1) {
+ return (b0 == null || b1 == null)
+ ? castNonNull(null)
+ : ULong.valueOf(
+ UnsignedType.toBigInteger(b0).divide(UnsignedType.toBigInteger(b1)));
+ }
+
public static byte checkedDivide(byte b0, byte b1) {
return intToByte(b0 / b1);
}
@@ -2742,6 +2859,22 @@ public class SqlFunctions {
}
}
+ public static UByte checkedDivide(UByte b0, UByte b1) {
+ return UByte.valueOf(b0.intValue() / b1.intValue());
+ }
+
+ public static UShort checkedDivide(UShort b0, UShort b1) {
+ return UShort.valueOf(b0.intValue() / b1.intValue());
+ }
+
+ public static UInteger checkedDivide(UInteger b0, UInteger b1) {
+ return UInteger.valueOf(b0.longValue() / b1.longValue());
+ }
+
+ public static ULong checkedDivide(ULong b0, ULong b1) {
+ return ULong.valueOf(UnsignedType.toBigInteger(b0).divide(UnsignedType.toBigInteger(b1)));
+ }
+
// *
/** SQL <code>*</code> operator applied to int values. */
@@ -2764,6 +2897,32 @@ public class SqlFunctions {
return (b0 == null || b1 == null) ? castNonNull(null) : (b0 * b1);
}
+ public static UByte multiply(UByte b0, UByte b1) {
+ return (b0 == null || b1 == null)
+ ? castNonNull(null)
+ : UByte.valueOf(b0.longValue() * b1.longValue());
+ }
+
+ public static UShort multiply(UShort b0, UShort b1) {
+ return (b0 == null || b1 == null)
+ ? castNonNull(null)
+ : UShort.valueOf(b0.intValue() * b1.intValue());
+ }
+
+ public static UInteger multiply(UInteger b0, UInteger b1) {
+ return (b0 == null || b1 == null)
+ ? castNonNull(null)
+ : UInteger.valueOf(b0.longValue() * b1.longValue());
+ }
+
+ public static ULong multiply(ULong b0, ULong b1) {
+ if (b0 == null || b1 == null) {
+ return castNonNull(null);
+ }
+ BigInteger result = UnsignedType.toBigInteger(b0).multiply(UnsignedType.toBigInteger(b1));
+ return ULong.valueOf(result);
+ }
+
/** SQL <code>*</code> operator applied to nullable long and int values. */
public static Long multiply(Long b0, Integer b1) {
return (b0 == null || b1 == null) ? castNonNull(null) : (b0.longValue() * b1.longValue());
@@ -2813,6 +2972,22 @@ public class SqlFunctions {
return Math.multiplyExact(b0, b1);
}
+ public static UByte checkedMultiply(UByte b0, UByte b1) {
+ return UByte.valueOf(b0.intValue() * b1.intValue());
+ }
+
+ public static UShort checkedMultiply(UShort b0, UShort b1) {
+ return UShort.valueOf(b0.intValue() * b1.intValue());
+ }
+
+ public static UInteger checkedMultiply(UInteger b0, UInteger b1) {
+ return UInteger.valueOf(b0.longValue() * b1.longValue());
+ }
+
+ public static ULong checkedMultiply(ULong b0, ULong b1) {
+ return ULong.valueOf(UnsignedType.toBigInteger(b0).multiply(UnsignedType.toBigInteger(b1)));
+ }
+
/** SQL <code>SAFE_ADD</code> function applied to long values. */
public static @Nullable Long safeAdd(long b0, long b1) {
try {
@@ -3105,6 +3280,126 @@ public class SqlFunctions {
return binaryOperator(b0, b1, (x, y) -> (byte) (x & y));
}
+ /**
+ * Bitwise function <code>BITAND</code> applied to {@link org.joou.UByte} values. Returns {@code
+ * null} if any operand is null.
+ */
+ public static UByte bitAnd(UByte b0, UByte b1) {
+ return UByte.valueOf((short) (b0.shortValue() & b1.shortValue()));
+ }
+
+ /**
+ * Bitwise function <code>BITAND</code> applied to {@link org.joou.UShort} values. Returns
+ * {@code null} if any operand is null.
+ */
+ public static UShort bitAnd(UShort b0, UShort b1) {
+ return UShort.valueOf(b0.intValue() & b1.intValue());
+ }
+
+ /**
+ * Bitwise function <code>BITAND</code> applied to {@link org.joou.UInteger} values. Returns
+ * {@code null} if any operand is null.
+ */
+ public static UInteger bitAnd(UInteger b0, UInteger b1) {
+ return UInteger.valueOf(b0.longValue() & b1.longValue());
+ }
+
+ /**
+ * Bitwise function <code>BITAND</code> applied to {@link org.joou.ULong} values. Returns {@code
+ * null} if any operand is null.
+ */
+ public static ULong bitAnd(ULong b0, ULong b1) {
+ return ULong.valueOf(b0.longValue() & b1.longValue());
+ }
+
+ /**
+ * Bitwise function <code>BITAND</code> applied to {@link org.joou.UInteger} and {@link Integer}
+ * values. Returns {@code null} if any operand is null.
+ */
+ public static long bitAnd(UInteger b0, long b1) {
+ return b0.intValue() & b1;
+ }
+
+ /**
+ * Bitwise function <code>BITAND</code> applied to {@link org.joou.ULong} and {@link long}
+ * values. Returns {@code null} if any operand is null.
+ */
+ public static long bitAnd(ULong b0, long b1) {
+ return b0.longValue() & b1;
+ }
+
+ /**
+ * Bitwise function <code>BITAND</code> applied to {@link long} and {@link org.joou.UInteger}
+ * values. Returns {@code null} if any operand is null.
+ */
+ public static long bitAnd(long b1, ULong b2) {
+ return b1 & b2.longValue();
+ }
+
+ /**
+ * Bitwise function <code>BITAND</code> applied to {@link long} and {@link org.joou.UInteger}
+ * values. Returns {@code null} if any operand is null.
+ */
+ public static long bitAnd(long b1, UInteger b2) {
+ return b1 & b2.longValue();
+ }
+
+ /**
+ * Bitwise function <code>BITAND</code> applied to {@link org.joou.UShort} and {@link Integer}
+ * values. Returns {@code null} if any operand is null.
+ */
+ public static long bitAnd(UShort b0, long b1) {
+ return b0.intValue() & b1;
+ }
+
+ /**
+ * Bitwise function <code>BITAND</code> applied to {@link org.joou.UShort} and {@link Integer}
+ * values. Returns {@code null} if any operand is null.
+ */
+ public static long bitAnd(long b0, UShort b1) {
+ return b0 & b1.intValue();
+ }
+
+ /**
+ * Bitwise function <code>BITAND</code> applied to {@link org.joou.UInteger} and {@link Integer}
+ * values. Returns {@code null} if any operand is null.
+ */
+ public static UInteger bitAnd(UInteger b0, int b1) {
+ return UInteger.valueOf(b0.intValue() & b1);
+ }
+
+ /**
+ * Bitwise function <code>BITAND</code> applied to {@link org.joou.UInteger} and {@link Integer}
+ * values. Returns {@code null} if any operand is null.
+ */
+ public static ULong bitAnd(ULong b0, int b1) {
+ return ULong.valueOf(b0.longValue() & b1);
+ }
+
+ /**
+ * Bitwise function <code>BITAND</code> applied to {@link Integer} and {@link org.joou.UInteger}
+ * values. Returns {@code null} if any operand is null.
+ */
+ public static ULong bitAnd(int b1, ULong b2) {
+ return ULong.valueOf(b1 & b2.longValue());
+ }
+
+ /**
+ * Bitwise function <code>BITAND</code> applied to {@link org.joou.UShort} and {@link Integer}
+ * values. Returns {@code null} if any operand is null.
+ */
+ public static Integer bitAnd(int b0, UShort b1) {
+ return b0 & b1.intValue();
+ }
+
+ /**
+ * Bitwise function <code>BITAND</code> applied to {@link org.joou.UShort} and {@link Integer}
+ * values. Returns {@code null} if any operand is null.
+ */
+ public static Integer bitAnd(UShort b0, int b1) {
+ return b0.intValue() & b1;
+ }
+
/**
* Helper function for implementing <code>BITCOUNT</code>. Counts the number of bits set in an
* integer value.
@@ -3232,16 +3527,16 @@ public class SqlFunctions {
}
/**
- * Bitwise function <code>BITXOR</code> applied to a Long and int value. Needed for handling
- * NULL for the first argument.
+ * Bitwise function <code>BITXOR</code> applied to a Long and int value. Overload to support
+ * type coercion between boxed Long and primitive int.
*/
public static long bitXor(Long b0, int b1) {
return b0 ^ b1;
}
/**
- * Bitwise function <code>BITXOR</code> applied to a Long and int value. Needed for handling
- * NULL for the second argument.
+ * Bitwise function <code>BITXOR</code> applied to a Long and int value. Overload to support
+ * type coercion between boxed Long and primitive int.
*/
public static long bitXor(int b0, Long b1) {
return b0 ^ b1;
@@ -3267,6 +3562,62 @@ public class SqlFunctions {
return new ByteString(result);
}
+ /**
+ * Bitwise function <code>BITXOR</code> applied to {@link Long} values. Returns {@code null} if
+ * any operand is null.
+ */
+ public static long bitXor(Long b0, Long b1) {
+ return b0 ^ b1;
+ }
+
+ /**
+ * Bitwise function <code>BITXOR</code> applied to {@link Integer} values. Returns {@code null}
+ * if any operand is null.
+ */
+ public static long bitXor(Integer b0, Integer b1) {
+ return b0 ^ b1;
+ }
+
+ /**
+ * Bitwise function <code>BITXOR</code> applied to {@link org.joou.UByte} values. Returns {@code
+ * null} if any operand is null.
+ */
+ public static UByte bitXor(UByte b0, UByte b1) {
+ return UByte.valueOf(b0.shortValue() ^ b1.shortValue());
+ }
+
+ /**
+ * Bitwise function <code>BITXOR</code> applied to {@link org.joou.UShort} values. Returns
+ * {@code null} if any operand is null.
+ */
+ public static UShort bitXor(UShort b0, UShort b1) {
+ return UShort.valueOf(b0.intValue() ^ b1.intValue());
+ }
+
+ /**
+ * Bitwise function <code>BITXOR</code> applied to {@link org.joou.UInteger} values. Returns
+ * {@code null} if any operand is null.
+ */
+ public static UInteger bitXor(UInteger b0, UInteger b1) {
+ return UInteger.valueOf(b0.longValue() ^ b1.longValue());
+ }
+
+ /**
+ * Bitwise function <code>BITXOR</code> applied to {@link org.joou.ULong} values. Returns {@code
+ * null} if any operand is null.
+ */
+ public static ULong bitXor(ULong b0, ULong b1) {
+ return ULong.valueOf(b0.longValue() ^ b1.longValue());
+ }
+
+ public static @Nullable Object bitXor(@Nullable Object b0, @Nullable Object b1) {
+ if (b0 == null || b1 == null) {
+ return null;
+ }
+ throw new IllegalArgumentException(
+ "Invalid arguments for BITXOR: " + "" + b0.getClass() + ", " + b1.getClass());
+ }
+
/**
* Utility for bitwise function applied to two byteString values.
*
@@ -3296,6 +3647,139 @@ public class SqlFunctions {
return new ByteString(result);
}
+ /**
+ * Performs PostgresSQL-style bitwise shift on a 32-bit integer.
+ *
+ * @param x the integer value to shift
+ * @param y the shift amount (positive: left shift, negative: right shift)
+ * @return the shifted integer
+ */
+ public static int leftShift(int x, int y) {
+ int shift = ((y % 32) + 32) % 32; // normalize to 0~31
+ return y >= 0 ? x << shift : x >> shift; // arithmetic right shift
+ }
+
+ // ----------------- long -----------------
+ /**
+ * Performs PostgresSQL-style bitwise shift on a 64-bit long value.
+ *
+ * @param x the long value to shift
+ * @param y the shift amount
+ * @return the shifted long value
+ */
+ public static long leftShift(long x, int y) {
+ int shift = ((y % 64) + 64) % 64; // normalize to 0~63
+ return y >= 0 ? x << shift : x >> shift;
+ }
+
+ /**
+ * Performs PostgresSQL-style bitwise shift on an int value with a long shift amount.
+ *
+ * @param x the int value to shift
+ * @param y the long shift amount
+ * @return the shifted value as long
+ */
+ public static long leftShift(int x, long y) {
+ int shift = (int) (((y % 32) + 32) % 32); // normalize to 0~31
+ return y >= 0 ? (long) x << shift : (long) x >> shift;
+ }
+
+ /**
+ * Performs PostgresSQL-style bitwise shift on a byte array. Positive shift: left shift.
+ * Negative shift: treated as positive shift with modulo arithmetic.
+ *
+ * @param bytes the input byte array
+ * @param y the shift amount in bits
+ * @return the shifted byte array
+ */
+ public static byte[] leftShift(byte[] bytes, int y) {
+ if (bytes.length == 0) {
+ return new byte[0];
+ }
+
+ int bitLen = bytes.length * 8;
+
+ // PostgreSQL behavior: always treat as left shift with modulo arithmetic
+ // Negative y becomes equivalent positive shift
+ int shift = ((y % bitLen) + bitLen) % bitLen;
+
+ if (shift == 0) {
+ return bytes.clone();
+ }
+
+ byte[] result = new byte[bytes.length];
+
+ // Always perform left shift (even for originally negative y)
+ int byteShift = shift / 8;
+ int bitShift = shift % 8;
+
+ for (int i = 0; i < bytes.length; i++) {
+ int srcIndex = i - byteShift;
+ int val = 0;
+
+ // Get the main byte
+ if (srcIndex >= 0) {
+ val = (bytes[srcIndex] & 0xFF) << bitShift;
+ }
+
+ // Get carry bits from previous byte
+ if (srcIndex - 1 >= 0 && bitShift != 0) {
+ val |= (bytes[srcIndex - 1] & 0xFF) >>> (8 - bitShift);
+ }
+
+ result[i] = (byte) val;
+ }
+ return result;
+ }
+
+ /**
+ * Performs PostgresSQL-style bitwise shift on ByteString.
+ *
+ * @param bytes the ByteString to shift
+ * @param y the shift amount in bits
+ * @return shifted ByteString
+ */
+ public static ByteString leftShift(ByteString bytes, int y) {
+ return new ByteString(leftShift(bytes.getBytes(), y));
+ }
+
+ /** Performs PostgresSQL-style bitwise shift on UByte. Overflow bits are masked to 8 bits. */
+ public static UByte leftShift(UByte x, int y) {
+ int shift = ((y % 8) + 8) % 8;
+ int val = x.byteValue() & 0xFF;
+ val = (y >= 0) ? (val << shift) & 0xFF : (val >> shift) & 0xFF;
+ return UByte.valueOf((byte) val);
+ }
+
+ /** Performs PostgresSQL-style bitwise shift on UShort. Overflow bits are masked to 16 bits. */
+ public static UShort leftShift(UShort x, int y) {
+ int shift = ((y % 16) + 16) % 16;
+ int val = x.shortValue() & 0xFFFF;
+ val = (y >= 0) ? (val << shift) & 0xFFFF : (val >> shift) & 0xFFFF;
+ return UShort.valueOf((short) val);
+ }
+
+ /**
+ * Performs PostgresSQL-style bitwise shift on UInteger. Overflow bits are masked to 32 bits.
+ */
+ public static UInteger leftShift(UInteger x, int y) {
+ int shift = ((y % 32) + 32) % 32;
+ long val = x.longValue() & 0xFFFFFFFFL;
+ val = (y >= 0) ? (val << shift) & 0xFFFFFFFFL : (val >> shift) & 0xFFFFFFFFL;
+ return UInteger.valueOf(val);
+ }
+
+ /**
+ * Performs PostgresSQL-style bitwise shift on ULong. Overflow bits are masked to 64 bits (long
+ * shifts naturally truncate).
+ */
+ public static ULong leftShift(ULong x, int y) {
+ int shift = ((y % 64) + 64) % 64;
+ long val = x.longValue();
+ val = (y >= 0) ? val << shift : val >> shift;
+ return ULong.valueOf(val);
+ }
+
// EXP
/** SQL <code>EXP</code> operator applied to double values. */
@@ -5870,7 +6354,7 @@ public class SqlFunctions {
return s.replace(search, replacement);
}
// for MSSQL's REPLACE function, search pattern is case-insensitive during matching
- return org.apache.commons.lang3.StringUtils.replaceIgnoreCase(s, search, replacement);
+ return org.apache.commons.lang3.Strings.CI.replace(s, search, replacement);
}
/**
@@ -6667,12 +7151,12 @@ public class SqlFunctions {
* array by splitting the input string value into subvalues using the specified string value as
* the "delimiter". Optionally, allows a specified string value to be interpreted as NULL.
*/
- public static List<String> stringToArray(String string, @Nullable String delimiter) {
+ public static List<@Nullable String> stringToArray(String string, @Nullable String delimiter) {
return stringToArray(string, delimiter, null);
}
/** SQL {@code STRING_TO_ARRAY(string, delimiter, nullString)} function. */
- public static List<String> stringToArray(
+ public static List<@Nullable String> stringToArray(
String string, @Nullable String delimiter, @Nullable String nullString) {
String[] parts;
if (delimiter == null) {
@@ -6685,7 +7169,7 @@ public class SqlFunctions {
} else {
parts = string.split(delimiter);
}
- List<String> result = new ArrayList<>(parts.length);
+ List<@Nullable String> result = new ArrayList<>(parts.length);
for (String part : parts) {
if (nullString != null && nullString.equals(part)) {
result.add(null);
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCoalesceFunction.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCoalesceFunction.java
index 556a0c2abd2..da666184d9e 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCoalesceFunction.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCoalesceFunction.java
@@ -123,11 +123,10 @@ public class SqlCoalesceFunction extends SqlFunction {
}
if (returnType == null) {
- throw new IllegalArgumentException(
- "Cannot infer return type for "
- + opBinding.getOperator()
- + "; operand types: "
- + opBinding.collectOperandTypes());
+ throw opBinding.newError(
+ RESOURCE.cannotInferReturnType(
+ opBinding.getOperator().toString(),
+ opBinding.collectOperandTypes().toString()));
}
return returnType;
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java
index 6bfc2cedf73..0f655a998b4 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java
@@ -19,6 +19,7 @@ package org.apache.calcite.sql.fun;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.sql.SqlAggFunction;
import org.apache.calcite.sql.SqlAsOperator;
import org.apache.calcite.sql.SqlBasicCall;
@@ -85,12 +86,13 @@ import java.util.function.Supplier;
import static java.util.Objects.requireNonNull;
import static org.apache.calcite.linq4j.Nullness.castNonNull;
+import static org.apache.calcite.util.Static.RESOURCE;
/**
* Implementation of {@link org.apache.calcite.sql.SqlOperatorTable} containing the standard
* operators and functions.
*
- * <p>Lines 822 ~ 824, 842 ~ 844, 853 ~ 855, 864 ~ 866, 875 ~ 877, 886 ~ 888, 897 ~ 899, Flink
+ * <p>Lines 828 ~ 830, 848 ~ 850, 859 ~ 861, 870 ~ 872, 881 ~ 883, 892 ~ 894, 903 ~ 905, Flink
* changes the return type of the {@code IS [NOT] JSON ...} predicates from {@link
* ReturnTypes#BOOLEAN_NULLABLE} to {@link ReturnTypes#BOOLEAN} so that they always return a
* non-nullable {@code BOOLEAN}, see also FLINK-39943.
@@ -258,10 +260,14 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable {
== SqlTypeFamily.ARRAY
? ReturnTypes.LEAST_RESTRICTIVE
: ReturnTypes.DYADIC_STRING_SUM_PRECISION_NULLABLE;
-
- return requireNonNull(
- returnType.inferReturnType(opBinding),
- "inferred CONCAT element type");
+ RelDataType type = returnType.inferReturnType(opBinding);
+ if (type == null) {
+ throw opBinding.newError(
+ RESOURCE.cannotInferReturnType(
+ opBinding.getOperator().toString(),
+ opBinding.collectOperandTypes().toString()));
+ }
+ return type;
}),
null,
OperandTypes.STRING_SAME_SAME_OR_ARRAY_SAME_SAME);
@@ -1167,6 +1173,42 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable {
ReturnTypes.LARGEST_INT_OR_FIRST_NON_NULL,
OperandTypes.INTEGER_INTEGER.or(OperandTypes.BINARY_BINARY));
+ /** <code>{@code ^}</code> operator. */
+ public static final SqlBinaryOperator BITXOR_OPERATOR =
+ new SqlBinaryOperator(
+ "^",
+ SqlKind.BITXOR,
+ 40,
+ true,
+ ReturnTypes.LARGEST_INT_OR_FIRST_NON_NULL,
+ InferTypes.FIRST_KNOWN,
+ OperandTypes.INTEGER_INTEGER
+ .or(OperandTypes.BINARY_BINARY)
+ .or(OperandTypes.UNSIGNED_NUMERIC_UNSIGNED_NUMERIC));
+
+ // Both operands should support bitwise operations
+
+ /** <code>{@code &}</code> operator. */
+ public static final SqlBinaryOperator BITAND_OPERATOR =
+ new SqlBinaryOperator(
+ "&",
+ SqlKind.BITAND,
+ 50, // Higher precedence than XOR but lower than multiplication
+ true,
+ ReturnTypes.LEAST_RESTRICTIVE,
+ InferTypes.FIRST_KNOWN,
+ OperandTypes.INTEGER_INTEGER
+ .or(OperandTypes.BINARY_BINARY)
+ .or(OperandTypes.UNSIGNED_NUMERIC_UNSIGNED_NUMERIC)
+ .or(
+ OperandTypes.family(
+ SqlTypeFamily.UNSIGNED_NUMERIC,
+ SqlTypeFamily.INTEGER)
+ .or(
+ OperandTypes.family(
+ SqlTypeFamily.INTEGER,
+ SqlTypeFamily.UNSIGNED_NUMERIC))));
+
/** <code>BITNOT</code> scalar function. */
public static final SqlFunction BITNOT =
SqlBasicFunction.create(
@@ -1184,6 +1226,33 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable {
/** <code>BIT_XOR</code> aggregate function. */
public static final SqlAggFunction BIT_XOR = new SqlBitOpAggFunction(SqlKind.BIT_XOR);
+ /** <code>{@code <<}</code> (left shift) operator. */
+ public static final SqlBinaryOperator BIT_LEFT_SHIFT =
+ new SqlBinaryOperator(
+ "<<",
+ SqlKind.OTHER,
+ 32, // Standard shift operator precedence
+ true,
+ ReturnTypes.ARG0_NULLABLE,
+ InferTypes.FIRST_KNOWN,
+ OperandTypes.or(
+ OperandTypes.family(SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER),
+ OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.INTEGER),
+ OperandTypes.family(
+ SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.INTEGER)));
+
+ /** left shift function. */
+ public static final SqlFunction LEFTSHIFT =
+ SqlBasicFunction.create(
+ "LEFTSHIFT",
+ SqlKind.OTHER_FUNCTION,
+ ReturnTypes.ARG0_NULLABLE,
+ OperandTypes.or(
+ OperandTypes.family(SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER),
+ OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.INTEGER),
+ OperandTypes.family(
+ SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.INTEGER)));
+
// -------------------------------------------------------------
// WINDOW Aggregate Functions
// -------------------------------------------------------------
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java
index 00f3c43fcb7..010b3afc893 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/type/SqlTypeFactoryImpl.java
@@ -40,8 +40,8 @@ import static org.apache.calcite.util.Static.RESOURCE;
* <p>FLINK modifications are at lines
*
* <ol>
- * <li>Should be removed after fixing CALCITE-6342: Lines 527-529.
- * <li>Should be removed after fix of FLINK-31350: Lines 604-616.
+ * <li>Should be removed after fixing CALCITE-6342: Lines 529-531.
+ * <li>Should be removed after fix of FLINK-31350: Lines 606-618.
* </ol>
*/
public class SqlTypeFactoryImpl extends RelDataTypeFactoryImpl {
@@ -327,6 +327,8 @@ public class SqlTypeFactoryImpl extends RelDataTypeFactoryImpl {
? createSqlType(typeName, type.getPrecision())
: createSqlType(typeName);
type = createTypeWithNullability(type, originalType.isNullable());
+ // update java type's family
+ family = type.getFamily();
}
if (resultType == null) {
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java
index 89d15cf6bd2..d33e27c81ab 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java
@@ -42,6 +42,7 @@ import org.apache.calcite.sql.validate.SqlNameMatcher;
import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.apache.calcite.sql.validate.SqlValidatorUtil;
+import org.apache.calcite.sql2rel.SqlToRelConverter;
import org.apache.calcite.util.NumberUtil;
import org.apache.calcite.util.Pair;
import org.apache.calcite.util.Util;
@@ -50,6 +51,7 @@ import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
import org.checkerframework.checker.nullness.qual.Nullable;
import java.math.BigDecimal;
+import java.math.BigInteger;
import java.math.RoundingMode;
import java.nio.charset.Charset;
import java.util.AbstractList;
@@ -75,9 +77,9 @@ import static org.apache.calcite.util.Static.RESOURCE;
* <p>FLINK modifications are at lines
*
* <ol>
- * <li>We should use ExtendedSqlCollectionTypeNameSpec for rows: Lines 1104-1112
- * <li>We should use ExtendedSqlRowTypeNameSpec for rows: Lines 1124-1128
- * <li>Should be removed after fixing CALCITE-7062: Lines 1148-1150
+ * <li>We should use ExtendedSqlCollectionTypeNameSpec for rows: Lines 1291-1300
+ * <li>We should use ExtendedSqlRowTypeNameSpec for rows: Lines 1312-1316
+ * <li>Should be removed after fixing CALCITE-7062: Lines 1336
* </ol>
*/
public abstract class SqlTypeUtil {
@@ -121,6 +123,40 @@ public abstract class SqlTypeUtil {
return true;
}
+ /**
+ * True if there are literals with the specified data type. Some data types do not have literals
+ * (e.g., UNSIGNED, ROW).
+ *
+ * @param type Type for literals.
+ */
+ public static boolean hasLiterals(RelDataType type) {
+ switch (type.getSqlTypeName()) {
+ case UTINYINT:
+ case USMALLINT:
+ case UINTEGER:
+ case UBIGINT:
+ case ANY:
+ case SYMBOL:
+ case MULTISET:
+ case ARRAY:
+ case MAP:
+ case DISTINCT:
+ case STRUCTURED:
+ case ROW:
+ case OTHER:
+ case CURSOR:
+ case COLUMN_LIST:
+ case DYNAMIC_STAR:
+ case GEOMETRY:
+ case MEASURE:
+ case FUNCTION:
+ case SARG:
+ return false;
+ default:
+ return true;
+ }
+ }
+
/**
* Returns whether the operands to a call are char type-comparable.
*
@@ -146,12 +182,58 @@ public abstract class SqlTypeUtil {
return true;
}
+ /**
+ * A namespace may contain multiple fields with the same name. However, a proper ROW type
+ * cannot; this function will assign unique names to fields when they are used to build a
+ * concrete ROW type.
+ *
+ * @param type A type for which type.isStruct() is true.
+ * @return A new version of this type where all fields have unique names.
+ * <p>Note: the same rule to rename fields is used by the {@link SqlToRelConverter} later,
+ * in convertNonAggregateSelectList (a private method). This ensures that the generated
+ * field names there will match with the inferred field names here.
+ */
+ private static RelDataType uniquify(RelDataTypeFactory factory, RelDataType type) {
+ List<String> unique = SqlValidatorUtil.uniquify(type.getFieldNames(), true);
+ List<RelDataType> types =
+ type.getFieldList().stream()
+ .map(RelDataTypeField::getType)
+ .collect(Collectors.toList());
+ return factory.createStructType(type.getStructKind(), types, unique);
+ }
+
+ /**
+ * Derives component type for ARRAY, MULTISET, MAP when input is sub-query.
+ *
+ * @param factory Type factory used to generate new types if necessary
+ * @param origin original component type
+ * @return component type
+ */
+ public static RelDataType deriveCollectionQueryComponentType(
+ RelDataTypeFactory factory, SqlTypeName collectionType, RelDataType origin) {
+ switch (collectionType) {
+ case ARRAY:
+ case MULTISET:
+ return origin.isStruct() && origin.getFieldCount() == 1
+ ? origin.getFieldList().get(0).getType()
+ : uniquify(factory, origin);
+ case MAP:
+ return origin;
+ default:
+ throw new AssertionError(
+ "Impossible to derive component type for " + collectionType);
+ }
+ }
+
/**
* Derives component type for ARRAY, MULTISET, MAP when input is sub-query.
*
* @param origin original component type
* @return component type
+ * @deprecated Use {@link SqlTypeUtil#deriveCollectionQueryComponentType(RelDataTypeFactory,
+ * SqlTypeName, RelDataType)}
*/
+ @Deprecated
public static RelDataType deriveCollectionQueryComponentType(
SqlTypeName collectionType, RelDataType origin) {
switch (collectionType) {
@@ -465,6 +547,10 @@ public abstract class SqlTypeUtil {
case SMALLINT:
case INTEGER:
case BIGINT:
+ case UTINYINT:
+ case USMALLINT:
+ case UINTEGER:
+ case UBIGINT:
return true;
default:
return false;
@@ -491,6 +577,10 @@ public abstract class SqlTypeUtil {
case SMALLINT:
case INTEGER:
case BIGINT:
+ case UTINYINT:
+ case USMALLINT:
+ case UINTEGER:
+ case UBIGINT:
case DECIMAL:
return true;
default:
@@ -498,12 +588,88 @@ public abstract class SqlTypeUtil {
}
}
+ /**
+ * Returns whether {@code container} can represent every value produced by {@code content}
+ * without loss of information.
+ *
+ * <p>The {@code container} type must be one of the integer types (signed or unsigned). The
+ * {@code content} type can be integer, or a DECIMAL with scale {@code 0}. For all other types
+ * this method returns {@code false}.
+ *
+ * @throws IllegalArgumentException if {@code container} is not an integer type
+ */
+ public static boolean integerRangeContains(RelDataType container, RelDataType content) {
+ checkArgument(isIntType(container), "container must be an integer type: %s", container);
+
+ final SqlTypeName contentType = content.getSqlTypeName();
+ final boolean contentIsDecimal = contentType == SqlTypeName.DECIMAL;
+ if (!isIntType(content) && (!contentIsDecimal || content.getScale() != 0)) {
+ return false;
+ }
+
+ final BigInteger containerMin = integerBound(container, false);
+ final BigInteger containerMax = integerBound(container, true);
+ if (containerMin == null || containerMax == null) {
+ return false;
+ }
+
+ final BigInteger contentMin = integerBound(content, false);
+ final BigInteger contentMax = integerBound(content, true);
+ if (contentMin == null || contentMax == null) {
+ return false;
+ }
+
+ return containerMin.compareTo(contentMin) <= 0 && containerMax.compareTo(contentMax) >= 0;
+ }
+
+ /**
+ * Returns the numeric bound for an integer or zero-scale decimal type.
+ *
+ * @param type Type whose bounds should be computed
+ * @param upper If {@code true}, returns the maximum inclusive bound; otherwise returns the
+ * minimum bound
+ * @return Bound as {@link BigInteger}, or {@code null} if the bound cannot be determined (for
+ * example, type is not integer or has non-zero scale)
+ */
+ public static @Nullable BigInteger integerBound(RelDataType type, boolean upper) {
+ final SqlTypeName typeName = type.getSqlTypeName();
+
+ final boolean isDecimal = typeName == SqlTypeName.DECIMAL;
+ if (!isDecimal && !isIntType(type)) {
+ return null;
+ }
+ if (isDecimal && type.getScale() != 0) {
+ return null;
+ }
+
+ final int precision = isDecimal ? type.getPrecision() : -1;
+ final int scale = isDecimal ? type.getScale() : -1;
+ final Object limit =
+ typeName.getLimit(upper, SqlTypeName.Limit.OVERFLOW, false, precision, scale);
+ if (limit == null) {
+ return null;
+ }
+ if (limit instanceof BigDecimal) {
+ try {
+ return ((BigDecimal) limit).toBigIntegerExact();
+ } catch (ArithmeticException ignored) {
+ return null;
+ }
+ }
+ if (limit instanceof Number) {
+ return BigInteger.valueOf(((Number) limit).longValue());
+ }
+ return null;
+ }
+
/** Returns whether a type's scale is set. */
public static boolean hasScale(RelDataType type) {
return type.getScale() != Integer.MIN_VALUE;
}
- /** Returns the maximum value of an integral type, as a long value. */
+ /**
+ * Returns the maximum value of an integral type, as a long value. DOES NOT WORK FOR UBIGINT.
+ */
public static long maxValue(RelDataType type) {
assert SqlTypeUtil.isIntType(type);
switch (type.getSqlTypeName()) {
@@ -513,6 +679,12 @@ public abstract class SqlTypeUtil {
return Short.MAX_VALUE;
case INTEGER:
return Integer.MAX_VALUE;
+ case UTINYINT:
+ return 255;
+ case USMALLINT:
+ return 65535;
+ case UINTEGER:
+ return (1L << 32) - 1;
case BIGINT:
return Long.MAX_VALUE;
default:
@@ -649,8 +821,9 @@ public abstract class SqlTypeUtil {
/**
* Returns the minimum unscaled value of a numeric type.
*
- * @param type a numeric type
+ * @deprecated Use {@link #integerBound(RelDataType, boolean)} with {@code upper = false}
*/
+ @Deprecated // to be removed before 2.0
public static long getMinValue(RelDataType type) {
SqlTypeName typeName = type.getSqlTypeName();
switch (typeName) {
@@ -660,6 +833,11 @@ public abstract class SqlTypeUtil {
return Short.MIN_VALUE;
case INTEGER:
return Integer.MIN_VALUE;
+ case UTINYINT:
+ case USMALLINT:
+ case UINTEGER:
+ case UBIGINT:
+ return 0;
case BIGINT:
case DECIMAL:
return NumberUtil.getMinUnscaled(type.getPrecision()).longValue();
@@ -669,17 +847,25 @@ public abstract class SqlTypeUtil {
}
/**
- * Returns the maximum unscaled value of a numeric type.
+ * Returns the maximum unscaled value of a numeric type. DOES NOT WORK CORRECTLY FOR U/BIGINT
+ * and many DECIMAL types.
*
- * @param type a numeric type
+ * @deprecated Use {@link #integerBound(RelDataType, boolean)} with {@code upper = true}
*/
+ @Deprecated // to be removed before 2.0
public static long getMaxValue(RelDataType type) {
SqlTypeName typeName = type.getSqlTypeName();
switch (typeName) {
+ case UTINYINT:
+ return 255;
case TINYINT:
return Byte.MAX_VALUE;
+ case USMALLINT:
+ return (1 << 16) - 1;
case SMALLINT:
return Short.MAX_VALUE;
+ case UINTEGER:
+ return (1L << 32) - 1;
case INTEGER:
return Integer.MAX_VALUE;
case BIGINT:
@@ -893,7 +1079,8 @@ public abstract class SqlTypeUtil {
return true;
}
if (toType.getSqlTypeName() == SqlTypeName.UUID) {
- return fromType.getSqlTypeName() == SqlTypeName.UUID
+ return fromType.getSqlTypeName() == SqlTypeName.NULL
+ || fromType.getSqlTypeName() == SqlTypeName.UUID
|| fromType.getFamily() == SqlTypeFamily.CHARACTER
|| fromType.getFamily() == SqlTypeFamily.BINARY;
}
@@ -1082,7 +1269,8 @@ public abstract class SqlTypeUtil {
if (isAtomic(type)
|| isNull(type)
|| type.getSqlTypeName() == SqlTypeName.UNKNOWN
- || type.getSqlTypeName() == SqlTypeName.GEOMETRY) {
+ || type.getSqlTypeName() == SqlTypeName.GEOMETRY
+ || SqlTypeUtil.isInterval(type)) {
int precision =
typeName.allowsPrec()
? type.getPrecision()
@@ -1145,9 +1333,7 @@ public abstract class SqlTypeUtil {
// REVIEW angel 11-Jan-2006:
// Use neg numbers to indicate unspecified precision/scale
- // FLINK MODIFICATION BEGIN
return new SqlDataTypeSpec(typeNameSpec, SqlParserPos.ZERO).withNullable(type.isNullable());
- // FLINK MODIFICATION END
}
/**
@@ -1464,7 +1650,6 @@ public abstract class SqlTypeUtil {
* @return Whether types are comparable
*/
public static boolean isComparable(RelDataType type1, RelDataType type2) {
- // FLINK MODIFICATION BEGIN Calcite-7230
final RelDataTypeFamily family1 = family(type1);
final RelDataTypeFamily family2 = family(type2);
@@ -1472,7 +1657,7 @@ public abstract class SqlTypeUtil {
if (family1 == SqlTypeFamily.NULL || family2 == SqlTypeFamily.NULL) {
return true;
}
- // FLINK MODIFICATION END
+
if (type1.isStruct() != type2.isStruct()) {
return false;
}
@@ -1522,11 +1707,6 @@ public abstract class SqlTypeUtil {
return true;
}
- // If one of the arguments is of type 'NULL', return true.
- if (family1 == SqlTypeFamily.NULL || family2 == SqlTypeFamily.NULL) {
- return true;
- }
-
// We can implicitly convert from character to date
return family1 == SqlTypeFamily.CHARACTER && canConvertStringInCompare(family2)
|| family2 == SqlTypeFamily.CHARACTER && canConvertStringInCompare(family1);
@@ -1813,7 +1993,8 @@ public abstract class SqlTypeUtil {
return SqlTypeUtil.isDatetime(type)
|| SqlTypeUtil.isNumeric(type)
|| SqlTypeUtil.isString(type)
- || SqlTypeUtil.isBoolean(type);
+ || SqlTypeUtil.isBoolean(type)
+ || typeName == SqlTypeName.UUID;
}
/** Returns a DECIMAL type with the maximum precision for the current type system. */
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
index da8bc0ba4a4..e8e9320718a 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
@@ -185,42 +185,40 @@ import static org.apache.calcite.util.Util.first;
* Default implementation of {@link SqlValidator}, the class was copied over because of
* CALCITE-4554.
*
- * <p>Lines 242 ~ 245, Flink improves error message for functions without appropriate arguments in
+ * <p>Lines 240 ~ 243, Flink improves error message for functions without appropriate arguments in
* handleUnresolvedFunction.
*
- * <p>Lines 1396 ~ 1398, CALCITE-7217, should be removed after upgrading Calcite to 1.41.0.
- *
- * <p>Lines 2171 ~ 2185, Flink improves error message for functions without appropriate arguments in
+ * <p>Lines 2127 ~ 2141, Flink improves error message for functions without appropriate arguments in
* handleUnresolvedFunction at {@link SqlValidatorImpl#handleUnresolvedFunction}.
*
- * <p>Lines 2440 ~ 2451
+ * <p>Lines 2396 ~ 2407
*
- * <p>Lines 2607 ~ 2609, CALCITE-7471 should be removed after upgrading Calcite to 1.42.0.
+ * <p>Lines 2563 ~ 2565, CALCITE-7471 should be removed after upgrading Calcite to 1.42.0.
*
- * <p>Lines 2723 ~ 2742, CALCITE-7217, CALCITE-7312 should be removed after upgrading Calcite to
+ * <p>Lines 2679 ~ 2698, CALCITE-7217, CALCITE-7312 should be removed after upgrading Calcite to
* 1.42.0.
*
- * <p>Lines 2773 ~ 2791, set the correct scope for VECTOR_SEARCH.
+ * <p>Lines 2729 ~ 2747, set the correct scope for VECTOR_SEARCH.
*
- * <p>Lines 4341 ~ 4345, 7193 ~ 7199 Flink improves Optimize the retrieval of sub-operands in
+ * <p>Lines 4320 ~ 4324, 7175 ~ 7181 Flink improves Optimize the retrieval of sub-operands in
* SqlCall when using NamedParameters at {@link SqlValidatorImpl#checkRollUp}.
*
- * <p>Lines 5923 ~ 5929, FLINK-24352 Add null check for temporal table check on SqlSnapshot.
+ * <p>Lines 5905 ~ 5911, FLINK-24352 Add null check for temporal table check on SqlSnapshot.
*
- * <p>Lines 6340-6355, CALCITE-7538 should be removed after upgrading Calcite to 1.42.0.
+ * <p>Lines 6322-6337, CALCITE-7538 should be removed after upgrading Calcite to 1.42.0.
*
- * <p>Lines 6365-6367, CALCITE-7466 should be removed after upgrading Calcite to 1.42.0.
+ * <p>Lines 6347-6349, CALCITE-7466 should be removed after upgrading Calcite to 1.42.0.
*
- * <p>Lines 6421-6423, CALCITE-7470 should be removed after upgrading Calcite to 1.42.0.
+ * <p>Lines 6403-6405, CALCITE-7470 should be removed after upgrading Calcite to 1.42.0.
*
- * <p>Lines 7284-7292, Added in FLINK-39695 (backport of CALCITE-6764): propagate parent record
+ * <p>Lines 7266-7274, Added in FLINK-39695 (backport of CALCITE-6764): propagate parent record
* nullability to nested fields.
*
- * <p>Lines 8025-8048, CALCITE-7486 should be removed after upgrading Calcite to 1.42.0.
+ * <p>Lines 8142-8165, CALCITE-7486 should be removed after upgrading Calcite to 1.42.0.
*
- * <p>Lines 8095-8112, CALCITE-7486 should be removed after upgrading Calcite to 1.42.0.
+ * <p>Lines 8212-8229, CALCITE-7486 should be removed after upgrading Calcite to 1.42.0.
*
- * <p>Lines 8157-8165, CALCITE-7486 should be removed after upgrading Calcite to 1.42.0.
+ * <p>Lines 8274-8282, CALCITE-7486 should be removed after upgrading Calcite to 1.42.0.
*/
public class SqlValidatorImpl implements SqlValidatorWithHints {
// ~ Static fields/initializers ---------------------------------------------
@@ -523,6 +521,15 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
// calls.
selectScope = getSelectScope(select);
expanded = expandSelectExpr(selectItem, scope, select, expansions);
+
+ // Non-strict GROUP BY: wrap non-aggregated, non-grouped columns in ANY_VALUE()
+ if (isAggregate(select)
+ && config.conformance().isNonStrictGroupBy()
+ && isNonAggregatedNonGroupedColumn(expanded, select)) {
+ expanded =
+ SqlStdOperatorTable.ANY_VALUE.createCall(
+ expanded.getParserPosition(), expanded);
+ }
}
final String alias = SqlValidatorUtil.alias(selectItem, aliases.size());
@@ -560,56 +567,32 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
return false;
}
- private static SqlNode expandExprFromJoin(
- SqlJoin join, SqlIdentifier identifier, SelectScope scope) {
- if (join.getConditionType() != JoinConditionType.USING) {
- return identifier;
+ /** Returns true if the node is a non-aggregated, non-grouped column in SELECT. */
+ private boolean isNonAggregatedNonGroupedColumn(SqlNode node, SqlSelect select) {
+ if (aggFinder.findAgg(node) != null) {
+ return false;
}
- final Map<String, String> fieldAliases = getFieldAliases(scope);
-
- for (String name : SqlIdentifier.simpleNames((SqlNodeList) getCondition(join))) {
- if (identifier.getSimple().equals(name)) {
- final List<SqlNode> qualifiedNode = new ArrayList<>();
- for (ScopeChild child : requireNonNull(scope, "scope").children) {
- if (child.namespace.getRowType().getFieldNames().contains(name)) {
- final SqlIdentifier exp =
- new SqlIdentifier(
- ImmutableList.of(child.name, name),
- identifier.getParserPosition());
- qualifiedNode.add(exp);
- }
- }
-
- assert qualifiedNode.size() == 2;
-
- // If there is an alias for the column, no need to wrap the coalesce with an AS
- // operator
- boolean haveAlias = fieldAliases.containsKey(name);
-
- final SqlCall coalesceCall =
- SqlStdOperatorTable.COALESCE.createCall(
- SqlParserPos.ZERO, qualifiedNode.get(0), qualifiedNode.get(1));
-
- if (haveAlias) {
- return coalesceCall;
- } else {
- return SqlStdOperatorTable.AS.createCall(
- SqlParserPos.ZERO,
- coalesceCall,
- new SqlIdentifier(name, SqlParserPos.ZERO));
- }
+ if (node instanceof SqlIdentifier) {
+ SqlNodeList groupList = select.getGroup();
+ if (groupList == null) {
+ return true;
}
+ return groupList.getList().stream()
+ .noneMatch(
+ groupItem ->
+ groupItem != null && node.equalsDeep(groupItem, Litmus.IGNORE));
}
- // Only need to try to expand the expr from the left input of join
- // since it is always left-deep join.
- final SqlNode node = join.getLeft();
- if (node instanceof SqlJoin) {
- return expandExprFromJoin((SqlJoin) node, identifier, scope);
- } else {
- return identifier;
+ if (node instanceof SqlCall) {
+ return ((SqlCall) node)
+ .getOperandList().stream()
+ .anyMatch(operand -> isNonAggregatedNonGroupedColumn(operand, select));
+ } else if (node instanceof SqlLiteral) {
+ return true;
}
+
+ return false;
}
private static Map<String, String> getFieldAliases(final SelectScope scope) {
@@ -661,31 +644,6 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
getNamespaceOrThrow(join.getRight()).getRowType());
}
- private static SqlNode expandCommonColumn(
- SqlSelect sqlSelect,
- SqlNode selectItem,
- SelectScope scope,
- SqlValidatorImpl validator) {
- if (!(selectItem instanceof SqlIdentifier)) {
- return selectItem;
- }
-
- final SqlNode from = sqlSelect.getFrom();
- if (!(from instanceof SqlJoin)) {
- return selectItem;
- }
-
- final SqlIdentifier identifier = (SqlIdentifier) selectItem;
- if (!identifier.isSimple()) {
- if (!validator.config().conformance().allowQualifyingCommonColumn()) {
- validateQualifiedCommonColumn((SqlJoin) from, identifier, scope, validator);
- }
- return selectItem;
- }
-
- return expandExprFromJoin((SqlJoin) from, identifier, scope);
- }
-
private static void validateQualifiedCommonColumn(
SqlJoin join, SqlIdentifier identifier, SelectScope scope, SqlValidatorImpl validator) {
List<String> names = validator.usingNames(join);
@@ -1393,9 +1351,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
}
// fall through
case TABLE_REF:
- // ----- FLINK MODIFICATION BEGIN -----
case LATERAL:
- // ----- FLINK MODIFICATION END -----
case SNAPSHOT:
case OVER:
case COLLECTION_TABLE:
@@ -3472,7 +3428,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
final BigDecimal noTrailingZeros = bd.stripTrailingZeros();
// If we don't strip trailing zeros we may reject values such as 1.000....0.
- final int maxPrecision = typeSystem.getMaxNumericPrecision();
+ final int maxPrecision = typeSystem.getMaxPrecision(SqlTypeName.DECIMAL);
if (noTrailingZeros.precision() > maxPrecision) {
throw newValidationError(
literal, RESOURCE.numberLiteralOutOfRange(bd.toString()));
@@ -3735,6 +3691,24 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
}
}
+ /**
+ * Get the number of scopes referenced by the specified node; the node represents a computation
+ * that will be converted to a Rel node eventually.
+ */
+ private int getScopeCount(SqlNode node) {
+ SqlValidatorScope scope = scopes.get(node);
+ if (scope == null) {
+ // Not all nodes have an associated scope; count these as "1".
+ // For example, a VALUES node.
+ return 1;
+ }
+ if (scope instanceof ListScope) {
+ ListScope join = (ListScope) scope;
+ return join.children.size();
+ }
+ return 1;
+ }
+
protected void validateJoin(SqlJoin join, SqlValidatorScope scope) {
final SqlNode left = join.getLeft();
final SqlNode right = join.getRight();
@@ -3836,9 +3810,11 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
condition, RESOURCE.asofConditionMustBeComparison());
}
+ int leftScopeCount = getScopeCount(left);
CompareFromBothSides validateCompare =
new CompareFromBothSides(
joinScope,
+ leftScopeCount,
catalogReader,
RESOURCE.asofConditionMustBeComparison());
condition.accept(validateCompare);
@@ -3858,7 +3834,10 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
// Change the exception in validateCompare when we validate the match condition
validateCompare =
new CompareFromBothSides(
- joinScope, catalogReader, RESOURCE.asofMatchMustBeComparison());
+ joinScope,
+ leftScopeCount,
+ catalogReader,
+ RESOURCE.asofMatchMustBeComparison());
matchCondition.accept(validateCompare);
break;
}
@@ -3874,16 +3853,21 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
*/
private class CompareFromBothSides extends SqlShuttle {
final SqlValidatorScope scope;
+ // Number of children scopes on the left side of the join.
+ // Used to determine whether an identifier is from the left input or the right input.
+ final int leftScopeCount;
final SqlValidatorCatalogReader catalogReader;
final Resources.ExInst<SqlValidatorException> exception;
private CompareFromBothSides(
SqlValidatorScope scope,
+ int leftScopeCount,
SqlValidatorCatalogReader catalogReader,
Resources.ExInst<SqlValidatorException> exception) {
this.scope = scope;
this.catalogReader = catalogReader;
this.exception = exception;
+ this.leftScopeCount = leftScopeCount;
}
@Override
@@ -3917,16 +3901,11 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
id.names.subList(0, id.names.size() - 1), nameMatcher, false, resolved);
SqlValidatorScope.Resolve resolve = resolved.only();
int index = resolve.path.steps().get(0).i;
- if (index == 0) {
+ if (index < leftScopeCount) {
leftFound = true;
- }
- if (index == 1) {
+ } else {
rightFound = true;
}
-
- if (!leftFound && !rightFound) {
- throw newValidationError(call, this.exception);
- }
}
if (!leftFound || !rightFound) {
// The comparison does not look at both tables
@@ -5686,13 +5665,6 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
// matched
boolean isUpdateModifiableViewTable = false;
if (query instanceof SqlUpdate) {
- final SqlNodeList targetColumnList =
- requireNonNull(((SqlUpdate) query).getTargetColumnList());
- final int targetColumnCount = targetColumnList.size();
- targetRowType =
- SqlTypeUtil.extractLastNFields(typeFactory, targetRowType, targetColumnCount);
- sourceRowType =
- SqlTypeUtil.extractLastNFields(typeFactory, sourceRowType, targetColumnCount);
isUpdateModifiableViewTable = table.unwrap(ModifiableViewTable.class) != null;
}
if (SqlTypeUtil.equalAsStructSansNullability(
@@ -5813,6 +5785,16 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
final RelDataType sourceRowType = getValidatedNodeType(select);
checkTypeAssignment(scopes.get(select), table, sourceRowType, targetRowType, call);
+ // Set validated sourceExpressionList from the source select.
+ // The last elements of sourceSelect are the expression list.
+ List<SqlNode> sourceExpressionList =
+ Util.last(select.getSelectList(), call.getSourceExpressionList().size());
+ call.setOperand(
+ 2,
+ SqlUtil.stripListAs(
+ new SqlNodeList(
+ sourceExpressionList,
+ call.getSourceExpressionList().getParserPosition())));
checkConstraint(table, call, targetRowType);
validateAccess(call.getTargetTable(), table, SqlAccessEnum.UPDATE);
@@ -7328,6 +7310,30 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
return requireNonNull(root.accept(this), () -> this + " returned null for " + root);
}
+ /** True if the exception ex indicates that name lookup has failed. */
+ public static boolean isNotFoundException(Exception ex) {
+ if (!(ex instanceof CalciteContextException)) {
+ return false;
+ }
+ String message = ex.getMessage();
+ if (message == null || !message.contains("not found")) {
+ return false;
+ }
+ return true;
+ }
+
+ /** True if the exception ex indicates that a column is ambiguous. */
+ public static boolean isAmbiguousException(Exception ex) {
+ if (!(ex instanceof CalciteContextException)) {
+ return false;
+ }
+ String message = ex.getMessage();
+ if (message == null || !message.contains("is ambiguous")) {
+ return false;
+ }
+ return true;
+ }
+
@Override
public @Nullable SqlNode visit(SqlIdentifier id) {
// First check for builtin functions which don't have
@@ -7383,6 +7389,88 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
}
return fqId;
}
+
+ protected SqlNode expandCommonColumn(
+ SqlSelect sqlSelect, SqlNode selectItem, SelectScope scope) {
+ if (!(selectItem instanceof SqlIdentifier)) {
+ return selectItem;
+ }
+
+ final SqlNode from = sqlSelect.getFrom();
+ if (!(from instanceof SqlJoin)) {
+ return selectItem;
+ }
+
+ final SqlIdentifier identifier = (SqlIdentifier) selectItem;
+ if (!identifier.isSimple()) {
+ if (!validator.config().conformance().allowQualifyingCommonColumn()) {
+ validateQualifiedCommonColumn((SqlJoin) from, identifier, scope, validator);
+ }
+ return selectItem;
+ }
+
+ return expandExprFromJoin((SqlJoin) from, identifier, scope);
+ }
+
+ private SqlNode expandExprFromJoin(
+ SqlJoin join, SqlIdentifier identifier, SelectScope scope) {
+ List<String> commonColumnNames;
+ // must be NATURAL or USING here, and cannot specify NATURAL keyword with USING clause
+ if (join.isNatural()) {
+ commonColumnNames = validator.deriveNaturalJoinColumnList(join);
+ } else if (join.getConditionType() == JoinConditionType.USING) {
+ commonColumnNames = SqlIdentifier.simpleNames((SqlNodeList) getCondition(join));
+ } else {
+ return identifier;
+ }
+
+ final SqlNameMatcher matcher = validator.getCatalogReader().nameMatcher();
+ final Map<String, String> fieldAliases = getFieldAliases(scope);
+
+ for (String name : commonColumnNames) {
+ if (matcher.matches(identifier.getSimple(), name)) {
+ final List<SqlNode> qualifiedNode = new ArrayList<>();
+ for (ScopeChild child : requireNonNull(scope, "scope").children) {
+ if (matcher.indexOf(child.namespace.getRowType().getFieldNames(), name)
+ >= 0) {
+ final SqlIdentifier exp =
+ new SqlIdentifier(
+ ImmutableList.of(child.name, name),
+ identifier.getParserPosition());
+ qualifiedNode.add(exp);
+ }
+ }
+
+ assert qualifiedNode.size() == 2;
+
+ // If there is an alias for the column, no need to wrap the coalesce with an AS
+ // operator
+ boolean haveAlias = fieldAliases.containsKey(name);
+
+ final SqlCall coalesceCall =
+ SqlStdOperatorTable.COALESCE.createCall(
+ SqlParserPos.ZERO, qualifiedNode.get(0), qualifiedNode.get(1));
+
+ if (haveAlias) {
+ return coalesceCall;
+ } else {
+ return SqlStdOperatorTable.AS.createCall(
+ SqlParserPos.ZERO,
+ coalesceCall,
+ new SqlIdentifier(identifier.getSimple(), SqlParserPos.ZERO));
+ }
+ }
+ }
+
+ // Only need to try to expand the expr from the left input of join
+ // since it is always left-deep join.
+ final SqlNode node = join.getLeft();
+ if (node instanceof SqlJoin) {
+ return expandExprFromJoin((SqlJoin) node, identifier, scope);
+ } else {
+ return identifier;
+ }
+ }
}
/**
@@ -7545,19 +7633,14 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
@Override
public @Nullable SqlNode visit(SqlIdentifier id) {
- final SqlNode node =
- expandCommonColumn(select, id, (SelectScope) getScope(), validator);
+ final SqlNode node = expandCommonColumn(select, id, (SelectScope) getScope());
if (node != id) {
return node;
} else {
try {
return super.visit(id);
} catch (Exception ex) {
- if (!(ex instanceof CalciteContextException)) {
- throw ex;
- }
- String message = ex.getMessage();
- if (message != null && !message.contains("not found")) {
+ if (!isNotFoundException(ex)) {
throw ex;
}
// This point is reached only if the name lookup failed using standard rules
@@ -7642,7 +7725,10 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
final SqlSelect select;
final SqlNode root;
final Clause clause;
- // Retain only expandable aliases or ordinals to prevent their expansion in a SQL call expr.
+ // This contains the ordinal nodes in a GROUP BY that may need to be expanded
+ // into column names when using a conformance that allows ordinals in group bys
+ // E.g., For GROUP BY 1, 1 will be in this list
+ // For GROUP BY CUBE(1), 1 will also be in this list
final Set<SqlNode> aliasOrdinalExpandSet = Sets.newIdentityHashSet();
ExtendedExpander(
@@ -7656,7 +7742,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
this.root = root;
this.clause = clause;
if (clause == Clause.GROUP_BY) {
- addExpandableExpressions();
+ addExpandableOrdinals();
}
}
@@ -7668,18 +7754,53 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
final SelectScope selectScope = validator.getRawSelectScopeNonNull(select);
final boolean replaceAliases = clause.shouldReplaceAliases(validator.config);
- if (!replaceAliases
- || (clause == Clause.GROUP_BY && !aliasOrdinalExpandSet.contains(id))) {
- SqlNode node = expandCommonColumn(select, id, selectScope, validator);
- if (node != id) {
- return node;
+ try {
+ // First try a standard expansion
+ if (clause == Clause.GROUP_BY) {
+ SqlNode node = expandCommonColumn(select, id, selectScope);
+ if (node != id) {
+ return node;
+ }
+ return super.visit(id);
}
- return super.visit(id);
+ } catch (Exception ex) {
+ // This behavior is from MySQL:
+ // - if there is no column in the FROM with the name used in the GROUP BY
+ // then look for a column alias in the SELECT
+ // - if there are multiple columns in the FROM with the name used in the GROUP BY,
+ // then also look for a column alias in the SELECT
+ if (!Expander.isNotFoundException(ex) && !Expander.isAmbiguousException(ex)) {
+ throw ex;
+ }
+ if (!replaceAliases) {
+ throw ex;
+ }
+ // Continue execution, trying to replace alias
}
+ final SqlNameMatcher nameMatcher = validator.catalogReader.nameMatcher();
+
+ if (clause == Clause.HAVING) {
+ if (!replaceAliases) {
+ return id;
+ }
+
+ // Do not expand aliases in HAVING if they are being grouped on
+ SqlNodeList list = select.getGroup();
+ if (list != null) {
+ // HAVING can be used without GROUP BY
+ for (SqlNode node : list) {
+ if (node instanceof SqlIdentifier) {
+ SqlIdentifier grouped = (SqlIdentifier) node;
+ if (nameMatcher.matches(id.getSimple(), Util.last(grouped.names))) {
+ return id;
+ }
+ }
+ }
+ }
+ }
String name = id.getSimple();
SqlNode expr = null;
- final SqlNameMatcher nameMatcher = validator.catalogReader.nameMatcher();
int n = 0;
for (SqlNode s : SqlNonNullableAccessors.getSelectList(select)) {
final @Nullable String alias = SqlValidatorUtil.alias(s);
@@ -7704,8 +7825,10 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
// expr cannot be null; in that case n = 0 would have returned
requireNonNull(expr, "expr");
- if (validator.getConformance().isSelectAlias()
- != SqlConformance.SelectAliasLookup.UNSUPPORTED) {
+ if ((clause == Clause.SELECT
+ && (validator.getConformance().isSelectAlias()
+ != SqlConformance.SelectAliasLookup.UNSUPPORTED))
+ || clause == Clause.GROUP_BY) {
Map<String, SqlNode> expansions = new HashMap<>();
final Expander expander =
new SelectExpander(validator, selectScope, select, expansions);
@@ -7751,18 +7874,13 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
break;
}
}
-
return super.visit(literal);
}
- /**
- * Add all possible expandable 'group by' expression to set, which is used to check whether
- * expr could be expanded as alias or ordinal.
- */
+ /** Add all possible expandable 'group by' ordinals to {@link aliasOrdinalExpandSet}. */
@RequiresNonNull({"root"})
- private void addExpandableExpressions() {
+ private void addExpandableOrdinals() {
switch (root.getKind()) {
- case IDENTIFIER:
case LITERAL:
aliasOrdinalExpandSet.add(root);
break;
@@ -7772,7 +7890,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
if (root instanceof SqlBasicCall) {
List<SqlNode> operandList = ((SqlBasicCall) root).getOperandList();
for (SqlNode sqlNode : operandList) {
- addIdentifierOrdinal2ExpandSet(sqlNode);
+ addOrdinal2ExpandSet(sqlNode);
}
}
break;
@@ -7782,18 +7900,17 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
}
/**
- * Identifier or literal in grouping sets, rollup, cube will be eligible for alias.
+ * Literal in grouping sets, rollup, cube will be expanded.
*
* @param sqlNode expression within grouping sets, rollup, cube
*/
- private void addIdentifierOrdinal2ExpandSet(SqlNode sqlNode) {
+ private void addOrdinal2ExpandSet(SqlNode sqlNode) {
if (sqlNode.getKind() == SqlKind.ROW) {
List<SqlNode> rowOperandList = ((SqlCall) sqlNode).getOperandList();
for (SqlNode node : rowOperandList) {
- addIdentifierOrdinal2ExpandSet(node);
+ addOrdinal2ExpandSet(node);
}
- } else if (sqlNode.getKind() == SqlKind.IDENTIFIER
- || sqlNode.getKind() == SqlKind.LITERAL) {
+ } else if (sqlNode.getKind() == SqlKind.LITERAL) {
aliasOrdinalExpandSet.add(sqlNode);
}
}
@@ -8361,14 +8478,10 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
boolean shouldReplaceAliases(Config config) {
switch (this) {
case GROUP_BY:
- return config.conformance().isGroupByAlias()
- || (config.conformance().isSelectAlias()
- != SqlConformance.SelectAliasLookup.UNSUPPORTED);
+ return config.conformance().isGroupByAlias();
case HAVING:
- return config.conformance().isHavingAlias()
- || (config.conformance().isSelectAlias()
- != SqlConformance.SelectAliasLookup.UNSUPPORTED);
+ return config.conformance().isHavingAlias();
case QUALIFY:
return true;
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
index 53e0c67f5c1..2a89f7862a9 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
@@ -57,7 +57,6 @@ import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.core.RelFactories;
import org.apache.calcite.rel.core.Sort;
-import org.apache.calcite.rel.core.Values;
import org.apache.calcite.rel.logical.LogicalAggregate;
import org.apache.calcite.rel.logical.LogicalCorrelate;
import org.apache.calcite.rel.logical.LogicalFilter;
@@ -112,11 +111,14 @@ import org.immutables.value.Value;
import org.slf4j.Logger;
import java.math.BigDecimal;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
@@ -136,9 +138,9 @@ import static org.apache.calcite.linq4j.Nullness.castNonNull;
* <p>TODO:
*
* <ol>
- * <li>Was changed within FLINK-29280, FLINK-28682, FLINK-35804: Line 247 ~ 254, Line 306 ~ 321
- * <li>Should be removed after fix of FLINK-29540: Line 326 ~ 332
- * <li>Should be removed after fix of FLINK-29540: Line 344 ~ 350
+ * <li>Was changed within FLINK-29280, FLINK-28682, FLINK-35804: Line 260 ~ 267, Line 319 ~ 334
+ * <li>Should be removed after fix of FLINK-29540: Line 339 ~ 345
+ * <li>Should be removed after fix of FLINK-29540: Line 357 ~ 363
* </ol>
*/
public class RelDecorrelator implements ReflectiveVisitor {
@@ -153,10 +155,21 @@ public class RelDecorrelator implements ReflectiveVisitor {
// map built during translation
protected CorelMap cm;
+ /**
+ * Stack maintaining visible Frames to the currently invoked RelNode during top-down traversal.
+ * Each entry maps a CorrelationId to the Frame where its correlated variables originate.
+ */
+ protected final Deque<Pair<CorrelationId, Frame>> frameStack = new ArrayDeque<>();
+
@SuppressWarnings("method.invocation.invalid")
protected final ReflectUtil.MethodDispatcher<@Nullable Frame> dispatcher =
ReflectUtil.<RelNode, @Nullable Frame>createMethodDispatcher(
- Frame.class, getVisitor(), "decorrelateRel", RelNode.class, boolean.class);
+ Frame.class,
+ getVisitor(),
+ "decorrelateRel",
+ RelNode.class,
+ boolean.class,
+ boolean.class);
// The rel which is being visited
protected @Nullable RelNode currentRel;
@@ -355,8 +368,28 @@ public class RelDecorrelator implements ReflectiveVisitor {
// Perform decorrelation.
map.clear();
- final Frame frame = getInvoke(root, false, null);
+ final Frame frame = getInvoke(root, false, null, true);
if (frame != null) {
+ // Check if the frame has more fields than the original and discard the extra ones
+ RelNode result = frame.r;
+ int fields = frame.r.getRowType().getFieldCount();
+ if (fields > frame.oldToNewOutputs.size()) {
+ relBuilder.push(result);
+ final List<RexNode> exprList = new ArrayList<>();
+ List<Map.Entry<Integer, Integer>> entries =
+ new ArrayList<>(frame.oldToNewOutputs.entrySet());
+ entries.sort(Map.Entry.comparingByKey());
+ for (Map.Entry<Integer, Integer> entry : entries) {
+ exprList.add(relBuilder.field(entry.getValue()));
+ }
+ relBuilder.project(exprList);
+ result = relBuilder.build();
+ } else {
+ Litmus.THROW.check(
+ fields == frame.oldToNewOutputs.size(),
+ "Produced relation has fewer columns than the original relation");
+ }
+
// has been rewritten; apply rules post-decorrelation
final HepProgramBuilder builder =
HepProgram.builder()
@@ -376,7 +409,7 @@ public class RelDecorrelator implements ReflectiveVisitor {
final HepProgram program2 = builder.build();
final HepPlanner planner2 = createPlanner(program2);
- final RelNode newRoot = frame.r;
+ final RelNode newRoot = result;
planner2.setRoot(newRoot);
return planner2.findBestExp();
}
@@ -504,14 +537,17 @@ public class RelDecorrelator implements ReflectiveVisitor {
}
/** Fallback if none of the other {@code decorrelateRel} methods match. */
- public @Nullable Frame decorrelateRel(RelNode rel, boolean isCorVarDefined) {
+ public @Nullable Frame decorrelateRel(
+ RelNode rel, boolean isCorVarDefined, boolean parentPropagatesNullValues) {
RelNode newRel = rel.copy(rel.getTraitSet(), rel.getInputs());
if (!rel.getInputs().isEmpty()) {
List<RelNode> oldInputs = rel.getInputs();
List<RelNode> newInputs = new ArrayList<>();
for (int i = 0; i < oldInputs.size(); ++i) {
- final Frame frame = getInvoke(oldInputs.get(i), isCorVarDefined, rel);
+ final Frame frame =
+ getInvoke(
+ oldInputs.get(i), isCorVarDefined, rel, parentPropagatesNullValues);
if (frame == null || !frame.corDefOutputs.isEmpty()) {
// if input is not rewritten, or if it produces correlated
// variables, terminate rewrite
@@ -535,7 +571,8 @@ public class RelDecorrelator implements ReflectiveVisitor {
ImmutableSortedMap.of());
}
- public @Nullable Frame decorrelateRel(Sort rel, boolean isCorVarDefined) {
+ public @Nullable Frame decorrelateRel(
+ Sort rel, boolean isCorVarDefined, boolean parentPropagatesNullValues) {
//
// Rewrite logic:
//
@@ -552,7 +589,7 @@ public class RelDecorrelator implements ReflectiveVisitor {
// need to call propagateExpr.
final RelNode oldInput = rel.getInput();
- final Frame frame = getInvoke(oldInput, isCorVarDefined, rel);
+ final Frame frame = getInvoke(oldInput, isCorVarDefined, rel, true);
if (frame == null) {
// If input has not been rewritten, do not rewrite this rel.
return null;
@@ -590,16 +627,13 @@ public class RelDecorrelator implements ReflectiveVisitor {
return register(rel, newSort, frame.oldToNewOutputs, frame.corDefOutputs);
}
- public @Nullable Frame decorrelateRel(Values rel, boolean isCorVarDefined) {
- // There are no inputs, so rel does not need to be changed.
- return null;
- }
-
- public @Nullable Frame decorrelateRel(LogicalAggregate rel, boolean isCorVarDefined) {
- return decorrelateRel((Aggregate) rel, isCorVarDefined);
+ public @Nullable Frame decorrelateRel(
+ LogicalAggregate rel, boolean isCorVarDefined, boolean parentPropagatesNullValues) {
+ return decorrelateRel((Aggregate) rel, isCorVarDefined, parentPropagatesNullValues);
}
- public @Nullable Frame decorrelateRel(Aggregate rel, boolean isCorVarDefined) {
+ public @Nullable Frame decorrelateRel(
+ Aggregate rel, boolean isCorVarDefined, boolean parentPropagatesNullValues) {
//
// Rewrite logic:
//
@@ -613,7 +647,7 @@ public class RelDecorrelator implements ReflectiveVisitor {
assert !cm.mapRefRelToCorRef.containsKey(rel);
final RelNode oldInput = rel.getInput();
- final Frame frame = getInvoke(oldInput, isCorVarDefined, rel);
+ final Frame frame = getInvoke(oldInput, isCorVarDefined, rel, parentPropagatesNullValues);
if (frame == null) {
// If input has not been rewritten, do not rewrite this rel.
return null;
@@ -648,7 +682,7 @@ public class RelDecorrelator implements ReflectiveVisitor {
}
// add mapping of group keys.
- outputMap.put(idx, newPos);
+ outputMap.put(i, newPos);
int newInputPos = requireNonNull(frame.oldToNewOutputs.get(idx));
RexInputRef.add2(projects, newInputPos, newInputOutput);
mapNewInputToProjOutputs.put(newInputPos, newPos);
@@ -762,9 +796,16 @@ public class RelDecorrelator implements ReflectiveVisitor {
combinedMap.get(oldAggCall.filterArg),
() -> "combinedMap.get(" + oldAggCall.filterArg + ")");
+ boolean newHasEmptyGroup = newGroupSets == null && newGroupSet.isEmpty();
+ if (newGroupSets != null) {
+ Iterator<ImmutableBitSet> groupSetsIterator = newGroupSets.iterator();
+ while (!newHasEmptyGroup && groupSetsIterator.hasNext()) {
+ newHasEmptyGroup |= groupSetsIterator.next().isEmpty();
+ }
+ }
newAggCalls.add(
oldAggCall.adaptTo(
- newProject, aggArgs, filterArg, oldGroupKeyCount, newGroupKeyCount));
+ newProject, aggArgs, filterArg, rel.hasEmptyGroup(), newHasEmptyGroup));
// The old to new output position mapping will be the same as that
// of newProject, plus any aggregates that the oldAgg produces.
@@ -796,11 +837,161 @@ public class RelDecorrelator implements ReflectiveVisitor {
RelNode newRel = relBuilder.build();
+ for (AggregateCall aggCall : rel.getAggCallList()) {
+ if (aggCall.getAggregation() instanceof SqlCountAggFunction) {
+ parentPropagatesNullValues = false;
+ break;
+ }
+ }
+
+ if (rel.getGroupType() == Aggregate.Group.SIMPLE
+ && rel.getGroupSet().isEmpty()
+ && !frame.corDefOutputs.isEmpty()
+ && !parentPropagatesNullValues) {
+ newRel = rewriteScalarAggregate(rel, newRel, outputMap, corDefOutputs);
+ }
+
// Aggregate does not change input ordering so corVars will be
// located at the same position as the input newProject.
return register(rel, newRel, outputMap, corDefOutputs);
}
+ /**
+ * Special case where the group by is static (i.e., aggregation functions without group by).
+ *
+ * <p>Background: For the query: SELECT SUM(salary), COUNT(name) FROM A; When table A is empty,
+ * it returns [null, 0]. But for SELECT SUM(salary), COUNT(name) FROM A group by id When table A
+ * is empty, it returns empty. This causes result mismatch. In the general decorrelation
+ * framework, we add corVar as an additional groupKey to rewrite Correlate as JOIN. (See the
+ * code above for details) This means that when the input is empty, the result produced using a
+ * JOIN is incorrect.
+ *
+ * <p>We refer to this situation as: `The well-known count bug`, More details about this issue:
+ * Optimization of Nested SQL Queries Revisited (https://dl.acm.org/doi/pdf/10.1145/38714.38723)
+ *
+ * <p>To handle this situation, we using a LEFT JOIN to ensure that an output is always
+ * produced.
+ *
+ * <p>Given the SQL: SELECT deptno FROM dept d WHERE 0 = (SELECT COUNT(*) FROM emp e WHERE
+ * d.deptno = e.deptno) Corresponding plan: LogicalProject(DEPTNO=[$0])
+ * LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}])
+ * LogicalProject(DEPTNO=[$0]) LogicalTableScan(table=[[scott, DEPT]]) LogicalProject(cs=[true])
+ * LogicalFilter(condition=[=(0, $0)]) LogicalAggregate(group=[{}], EXPR$0=[COUNT()])
+ * LogicalFilter(condition=[=($cor0.DEPTNO, $7)]) LogicalTableScan(table=[[scott, EMP]])
+ *
+ * <p>Rewriting this as: SELECT d.deptno FROM dept d JOIN ( SELECT true, e.deptno FROM emp e
+ * WHERE e.deptno IS NOT NULL GROUP BY e.deptno HAVING COUNT(*) = 0 ) AS d0 ON d.deptno =
+ * d0.deptno produces an incorrect result. Corresponding plan: LogicalProject(DEPTNO=[$0])
+ * LogicalJoin(condition=[=($0, $2)], joinType=[inner]) LogicalProject(DEPTNO=[$0])
+ * LogicalTableScan(table=[[scott, DEPT]]) LogicalProject(cs=[true], DEPTNO=[$0])
+ * LogicalFilter(condition=[=(0, $1)]) LogicalAggregate(group=[{0}], EXPR$0=[COUNT()]) //
+ * corresponds to {@code oldRel} LogicalProject(DEPTNO=[$7]) LogicalFilter(condition=[IS NOT
+ * NULL($7)]) LogicalTableScan(table=[[scott, EMP]]) We can clearly observe that due to the
+ * presence of the GROUP BY clause, COUNT(*) = 0 will never evaluate to true, since rows with
+ * zero records won't appear in the GROUP BY results. This produced incorrect results.
+ *
+ * <p>Rewrite Aggregate as: SELECT d.deptno FROM dept d JOIN ( SELECT true AS cs, deptno FROM (
+ * SELECT d2.deptno, CASE WHEN cnt0 IS NOT NULL THEN cnt0 ELSE 0 END AS cnt FROM (SELECT deptno
+ * FROM dept GROUP BY deptno) d2 LEFT JOIN ( SELECT deptno, COUNT(e.empno) cnt0 FROM emp WHERE
+ * deptno IS NOT NULL GROUP BY deptno) e ON d2.deptno IS NOT DISTINCT FROM e.deptno ) AS
+ * case_count WHERE cnt = 0 ) AS d0 ON d.deptno = d0.deptno Corresponding plan: [01]
+ * LogicalProject(DEPTNO=[$0]) [02] LogicalJoin(condition=[=($0, $2)], joinType=[inner]) [03]
+ * LogicalProject(DEPTNO=[$0]) [04] LogicalTableScan(table=[[scott, DEPT]]) [05]
+ * LogicalProject(cs=[true], DEPTNO=[$0]) [06] LogicalFilter(condition=[=(0, $1)]) [07]
+ * LogicalProject(DEPTNO=[$0], EXPR$0=[CASE(IS NOT NULL($2), $2, 0)]) [08]
+ * LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) [09]
+ * LogicalAggregate(group=[{0}]) [10] LogicalProject(DEPTNO=[$0]) [11]
+ * LogicalTableScan(table=[[scott, DEPT]]) [12] LogicalAggregate(group=[{0}], EXPR$0=[COUNT()])
+ * [13] LogicalProject(DEPTNO=[$7]) [14] LogicalFilter(condition=[IS NOT NULL($7)]) [15]
+ * LogicalTableScan(table=[[scott, EMP]])
+ *
+ * <p>Here we perform an early join, preserving all possible CorVar sets from the outer scope
+ * and their corresponding aggregation results. This ensures that for any row from the left
+ * input of the Correlation, there is always an aggregation result available for join output.
+ *
+ * <p>Implementation based on: Improving Unnesting of Complex Queries
+ * (https://dl.gi.de/server/api/core/bitstreams/c1918e8c-6a87-4da2-930a-bfed289f2388/content)
+ */
+ private RelNode rewriteScalarAggregate(
+ Aggregate oldRel,
+ RelNode newRel,
+ Map<Integer, Integer> outputMap,
+ NavigableMap<CorDef, Integer> corDefOutputs) {
+ final Pair<CorrelationId, Frame> outerFramePair = requireNonNull(this.frameStack.peek());
+ final Frame outFrame = outerFramePair.right;
+ RexBuilder rexBuilder = relBuilder.getRexBuilder();
+
+ int groupKeySize =
+ (int)
+ corDefOutputs.keySet().stream()
+ .filter(a -> a.corr.equals(outerFramePair.left))
+ .count();
+ List<RelDataTypeField> newRelFields = newRel.getRowType().getFieldList();
+ ImmutableBitSet.Builder corFieldBuilder = ImmutableBitSet.builder();
+
+ // Here we record the mapping between the original index and the new project.
+ // For the count, we map it as `case when x is null then 0 else x`.
+ final Map<Integer, RexNode> newProjectMap = new HashMap<>();
+ final List<RexNode> conditions = new ArrayList<>();
+ for (Map.Entry<CorDef, Integer> corDefOutput : corDefOutputs.entrySet()) {
+ CorDef corDef = corDefOutput.getKey();
+ Integer corIndex = corDefOutput.getValue();
+ if (corDef.corr.equals(outerFramePair.left)) {
+ int newIdx = requireNonNull(outFrame.oldToNewOutputs.get(corDef.field));
+ corFieldBuilder.set(newIdx);
+
+ RelDataType type = outFrame.r.getRowType().getFieldList().get(newIdx).getType();
+ RexNode left = new RexInputRef(corFieldBuilder.cardinality() - 1, type);
+ newProjectMap.put(corIndex + groupKeySize, left);
+ conditions.add(
+ relBuilder.isNotDistinctFrom(
+ left,
+ new RexInputRef(
+ corIndex + groupKeySize,
+ newRelFields.get(corIndex).getType())));
+ }
+ }
+
+ ImmutableBitSet groupSet = corFieldBuilder.build();
+ // Build [09] LogicalAggregate(group=[{0}]) to obtain the distinct set of
+ // corVar from outFrame.
+ relBuilder.push(outFrame.r).aggregate(relBuilder.groupKey(groupSet));
+
+ // Build [08] LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])
+ // to ensure each corVar's aggregate result is output.
+ final RelNode join = relBuilder.push(newRel).join(JoinRelType.LEFT, conditions).build();
+
+ for (int i1 = 0; i1 < oldRel.getAggCallList().size(); i1++) {
+ AggregateCall aggCall = oldRel.getAggCallList().get(i1);
+ if (aggCall.getAggregation() instanceof SqlCountAggFunction) {
+ int index = requireNonNull(outputMap.get(i1 + oldRel.getGroupSet().size()));
+ final RexInputRef ref = RexInputRef.of(index + groupKeySize, join.getRowType());
+ RexNode specificCountValue =
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.CASE,
+ ImmutableList.of(
+ relBuilder.isNotNull(ref), ref, relBuilder.literal(0)));
+ newProjectMap.put(ref.getIndex(), specificCountValue);
+ }
+ }
+
+ final List<RexNode> newProjects = new ArrayList<>();
+ for (int index : ImmutableBitSet.range(groupKeySize, join.getRowType().getFieldCount())) {
+ if (newProjectMap.containsKey(index)) {
+ newProjects.add(requireNonNull(newProjectMap.get(index)));
+ } else {
+ newProjects.add(RexInputRef.of(index, join.getRowType()));
+ }
+ }
+
+ // Build [07] LogicalProject(DEPTNO=[$0], EXPR$0=[CASE(IS NOT NULL($2), $2, 0)])
+ // to handle COUNT function by converting nulls to zero.
+ return relBuilder
+ .push(join)
+ .project(newProjects, newRel.getRowType().getFieldNames())
+ .build();
+ }
+
/**
* Shift the mapping to fixed offset from the {@code startIndex}.
*
@@ -817,8 +1008,18 @@ public class RelDecorrelator implements ReflectiveVisitor {
}
}
- public @Nullable Frame getInvoke(RelNode r, boolean isCorVarDefined, @Nullable RelNode parent) {
- final Frame frame = dispatcher.invoke(r, isCorVarDefined);
+ /**
+ * Invokes decorrelation logic for a given relational expression.
+ *
+ * @param parentPropagatesNullValues True if the parent RelNode produces null when all of its
+ * inputs fields are null.
+ */
+ public @Nullable Frame getInvoke(
+ RelNode r,
+ boolean isCorVarDefined,
+ @Nullable RelNode parent,
+ boolean parentPropagatesNullValues) {
+ final Frame frame = dispatcher.invoke(r, isCorVarDefined, parentPropagatesNullValues);
currentRel = parent;
if (frame != null) {
map.put(r, frame);
@@ -985,19 +1186,26 @@ public class RelDecorrelator implements ReflectiveVisitor {
return null;
}
- public @Nullable Frame decorrelateRel(LogicalProject rel, boolean isCorVarDefined) {
- return decorrelateRel((Project) rel, isCorVarDefined);
+ public @Nullable Frame decorrelateRel(
+ LogicalProject rel, boolean isCorVarDefined, boolean parentPropagatesNullValues) {
+ return decorrelateRel((Project) rel, isCorVarDefined, parentPropagatesNullValues);
}
- public @Nullable Frame decorrelateRel(Project rel, boolean isCorVarDefined) {
+ public @Nullable Frame decorrelateRel(
+ Project rel, boolean isCorVarDefined, boolean parentPropagatesNullValues) {
//
// Rewrite logic:
//
// 1. Pass along any correlated variables coming from the input.
//
+ for (RexNode project : rel.getProjects()) {
+ if (!Strong.isStrong(project)) {
+ parentPropagatesNullValues = false;
+ }
+ }
final RelNode oldInput = rel.getInput();
- Frame frame = getInvoke(oldInput, isCorVarDefined, rel);
+ Frame frame = getInvoke(oldInput, isCorVarDefined, rel, parentPropagatesNullValues);
if (frame == null) {
// If input has not been rewritten, do not rewrite this rel.
return null;
@@ -1361,25 +1569,31 @@ public class RelDecorrelator implements ReflectiveVisitor {
&& type.getPrecision() >= type1.getPrecision();
}
- public @Nullable Frame decorrelateRel(LogicalSnapshot rel, boolean isCorVarDefined) {
+ public @Nullable Frame decorrelateRel(
+ LogicalSnapshot rel, boolean isCorVarDefined, boolean parentPropagatesNullValues) {
if (RexUtil.containsCorrelation(rel.getPeriod())) {
return null;
}
- return decorrelateRel((RelNode) rel, isCorVarDefined);
+ return decorrelateRel((RelNode) rel, isCorVarDefined, parentPropagatesNullValues);
}
- public @Nullable Frame decorrelateRel(LogicalTableFunctionScan rel, boolean isCorVarDefined) {
+ public @Nullable Frame decorrelateRel(
+ LogicalTableFunctionScan rel,
+ boolean isCorVarDefined,
+ boolean parentPropagatesNullValues) {
if (RexUtil.containsCorrelation(rel.getCall())) {
return null;
}
- return decorrelateRel((RelNode) rel, isCorVarDefined);
+ return decorrelateRel((RelNode) rel, isCorVarDefined, parentPropagatesNullValues);
}
- public @Nullable Frame decorrelateRel(LogicalFilter rel, boolean isCorVarDefined) {
- return decorrelateRel((Filter) rel, isCorVarDefined);
+ public @Nullable Frame decorrelateRel(
+ LogicalFilter rel, boolean isCorVarDefined, boolean parentPropagatesNullValues) {
+ return decorrelateRel((Filter) rel, isCorVarDefined, parentPropagatesNullValues);
}
- public @Nullable Frame decorrelateRel(Filter rel, boolean isCorVarDefined) {
+ public @Nullable Frame decorrelateRel(
+ Filter rel, boolean isCorVarDefined, boolean parentPropagatesNullValues) {
//
// Rewrite logic:
//
@@ -1397,7 +1611,7 @@ public class RelDecorrelator implements ReflectiveVisitor {
//
final RelNode oldInput = rel.getInput();
- Frame frame = getInvoke(oldInput, isCorVarDefined, rel);
+ Frame frame = getInvoke(oldInput, isCorVarDefined, rel, parentPropagatesNullValues);
if (frame == null) {
// If input has not been rewritten, do not rewrite this rel.
return null;
@@ -1428,11 +1642,13 @@ public class RelDecorrelator implements ReflectiveVisitor {
return register(rel, relBuilder.build(), frame.oldToNewOutputs, frame.corDefOutputs);
}
- public @Nullable Frame decorrelateRel(LogicalCorrelate rel, boolean isCorVarDefined) {
- return decorrelateRel((Correlate) rel, isCorVarDefined);
+ public @Nullable Frame decorrelateRel(
+ LogicalCorrelate rel, boolean isCorVarDefined, boolean parentPropagatesNullValues) {
+ return decorrelateRel((Correlate) rel, isCorVarDefined, parentPropagatesNullValues);
}
- public @Nullable Frame decorrelateRel(Correlate rel, boolean isCorVarDefined) {
+ public @Nullable Frame decorrelateRel(
+ Correlate rel, boolean isCorVarDefined, boolean parentPropagatesNullValues) {
//
// Rewrite logic:
//
@@ -1446,15 +1662,18 @@ public class RelDecorrelator implements ReflectiveVisitor {
final RelNode oldLeft = rel.getInput(0);
final RelNode oldRight = rel.getInput(1);
- final Frame leftFrame = getInvoke(oldLeft, isCorVarDefined, rel);
- final Frame rightFrame = getInvoke(oldRight, true, rel);
-
- if (leftFrame == null || rightFrame == null) {
- // If any input has not been rewritten, do not rewrite this rel.
+ final Frame leftFrame =
+ getInvoke(oldLeft, isCorVarDefined, rel, parentPropagatesNullValues);
+ if (leftFrame == null) {
+ // If input has not been rewritten, do not rewrite this rel.
return null;
}
- if (rightFrame.corDefOutputs.isEmpty()) {
+ frameStack.push(Pair.of(rel.getCorrelationId(), leftFrame));
+ final Frame rightFrame = getInvoke(oldRight, true, rel, parentPropagatesNullValues);
+ frameStack.pop();
+
+ if (rightFrame == null || rightFrame.corDefOutputs.isEmpty()) {
return null;
}
@@ -1542,16 +1761,18 @@ public class RelDecorrelator implements ReflectiveVisitor {
return register(rel, newJoin, mapOldToNewOutputs, corDefOutputs);
}
- public @Nullable Frame decorrelateRel(LogicalJoin rel, boolean isCorVarDefined) {
- return decorrelateRel((Join) rel, isCorVarDefined);
+ public @Nullable Frame decorrelateRel(
+ LogicalJoin rel, boolean isCorVarDefined, boolean parentPropagatesNullValues) {
+ return decorrelateRel((Join) rel, isCorVarDefined, parentPropagatesNullValues);
}
- public @Nullable Frame decorrelateRel(Join rel, boolean isCorVarDefined) {
+ public @Nullable Frame decorrelateRel(
+ Join rel, boolean isCorVarDefined, boolean parentPropagatesNullValues) {
// For SEMI/ANTI join decorrelate it's input directly,
// because the correlate variables can only be propagated from
// the left side, which is not supported yet.
if (!rel.getJoinType().projectsRight()) {
- return decorrelateRel((RelNode) rel, isCorVarDefined);
+ return decorrelateRel((RelNode) rel, isCorVarDefined, parentPropagatesNullValues);
}
//
// Rewrite logic:
@@ -1563,8 +1784,10 @@ public class RelDecorrelator implements ReflectiveVisitor {
final RelNode oldLeft = rel.getInput(0);
final RelNode oldRight = rel.getInput(1);
- final Frame leftFrame = getInvoke(oldLeft, isCorVarDefined, rel);
- final Frame rightFrame = getInvoke(oldRight, isCorVarDefined, rel);
+ final Frame leftFrame =
+ getInvoke(oldLeft, isCorVarDefined, rel, parentPropagatesNullValues);
+ final Frame rightFrame =
+ getInvoke(oldRight, isCorVarDefined, rel, parentPropagatesNullValues);
if (leftFrame == null || rightFrame == null) {
// If any input has not been rewritten, do not rewrite this rel.
@@ -2764,8 +2987,8 @@ public class RelDecorrelator implements ReflectiveVisitor {
joinOutputProject,
argList,
filterArg,
- aggregate.getGroupCount(),
- groupCount));
+ aggregate.hasEmptyGroup(),
+ groupCount == 0));
}
ImmutableBitSet groupSet = ImmutableBitSet.range(groupCount);
@@ -2876,7 +3099,7 @@ public class RelDecorrelator implements ReflectiveVisitor {
onMatch2(d, call, correlate, left, aggOutputProject, aggregate);
}
- private void onMatch2(
+ private static void onMatch2(
RelDecorrelator d,
RelOptRuleCall call,
Correlate correlate,
@@ -3297,6 +3520,8 @@ public class RelDecorrelator implements ReflectiveVisitor {
Litmus.THROW);
assert allLessThan(
this.oldToNewOutputs.values(), r.getRowType().getFieldCount(), Litmus.THROW);
+ RelDataType rowType = oldRel.getRowType();
+ assert this.oldToNewOutputs.size() >= rowType.getFieldCount();
}
}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 84d9f3d274c..a1cb5e56cb5 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -49,6 +49,7 @@ import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelCollationTraitDef;
import org.apache.calcite.rel.RelCollations;
import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelHomogeneousShuttle;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelRoot;
import org.apache.calcite.rel.RelShuttleImpl;
@@ -101,6 +102,7 @@ import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexPatternFieldRef;
import org.apache.calcite.rex.RexRangeRef;
import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexSimplify;
import org.apache.calcite.rex.RexSubQuery;
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.rex.RexWindowBound;
@@ -262,21 +264,21 @@ import static org.apache.calcite.util.Util.transform;
* <p>FLINK modifications are at lines
*
* <ol>
- * <li>Added in FLINK-29081, FLINK-28682, FLINK-33395: Lines 709 ~ 726
- * <li>Added in FLINK-24024: Lines 1514 ~ 1520
- * <li>Added in FLINK-24024: Lines 1534 ~ 1573
- * <li>Added in FLINK-37269: Lines 2296 ~ 2318
- * <li>Added in FLINK-28682: Lines 2469 ~ 2486
- * <li>Added in FLINK-28682: Lines 2523 ~ 2551
- * <li>Added in FLINK-32474: Lines 2608 ~ 2610
- * <li>Added in FLINK-32474: Lines 2614 ~ 2616
- * <li>Added in FLINK-32474: Lines 2632 ~ 2634
- * <li>Added in CALCITE-7217: Lines 2673 ~ 2681, should be dropped with upgrade to Calcite 1.41.0
- * <li>Added in FLINK-32474: Lines 3075 ~ 3087
- * <li>Added in FLINK-32474: Lines 3188 ~ 3222
- * <li>Added in FLINK-38720: Lines 4823 ~ 4829, 4832 ~ 4836
- * <li>Added in FLINK-38720: Lines 4842 ~ 4917
- * <li>Added in FLINK-34312: Lines 6235 ~ 6246
+ * <li>Added in FLINK-29081, FLINK-28682, FLINK-33395: Lines 711 ~ 728
+ * <li>Added in FLINK-24024: Lines 1516 ~ 1522
+ * <li>Added in FLINK-24024: Lines 1536 ~ 1575
+ * <li>Added in FLINK-37269: Lines 2340 ~ 2362
+ * <li>Added in FLINK-28682: Lines 2513 ~ 2530
+ * <li>Added in FLINK-28682: Lines 2567 ~ 2595
+ * <li>Added in FLINK-32474: Lines 2652 ~ 2654
+ * <li>Added in FLINK-32474: Lines 2658 ~ 2660
+ * <li>Added in FLINK-32474: Lines 2676 ~ 2678
+ * <li>Added in FLINK-32474: Lines 3119 ~ 3131
+ * <li>Added in FLINK-32474: Lines 3232 ~ 3266
+ * <li>Added in CALCITE-7589: Lines 3756 ~ 3781
+ * <li>Added in FLINK-39695: Lines 4880 ~ 4886, 4889 ~ 4893
+ * <li>Added in FLINK-39695: Lines 4899 ~ 4974
+ * <li>Added in FLINK-34312: Lines 6297 ~ 6308
* </ol>
*
* <p>In official extension point (i.e. {@link #convertExtendedExpression(SqlNode, Blackboard)}):
@@ -1168,6 +1170,16 @@ public class SqlToRelConverter {
return e;
}
+ private RexNode simplifyPredicate(RexNode predicate) {
+ final RexNode converted = RexUtil.removeNullabilityCast(typeFactory, predicate);
+ List<RexNode> conjuncts = RelOptUtil.conjunctions(converted);
+ List<RexNode> simplified =
+ conjuncts.stream()
+ .map(e -> RexSimplify.simplifyComparisonWithNull(e, rexBuilder))
+ .collect(Collectors.toList());
+ return RexUtil.composeConjunction(rexBuilder, simplified);
+ }
+
/**
* Converts a WHERE clause.
*
@@ -1181,7 +1193,7 @@ public class SqlToRelConverter {
SqlNode newWhere = pushDownNotForIn(bb.scope, where);
replaceSubQueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE);
final RexNode convertedWhere = bb.convertExpression(newWhere);
- final RexNode convertedWhere2 = RexUtil.removeNullabilityCast(typeFactory, convertedWhere);
+ final RexNode convertedWhere2 = simplifyPredicate(convertedWhere);
// only allocate filter if the condition is not TRUE
if (convertedWhere2.isAlwaysTrue()) {
@@ -1284,23 +1296,8 @@ public class SqlToRelConverter {
if (!config.isExpand()) {
if (query instanceof SqlNodeList) {
- // convert
- // select * from "scott".emp where sal > some (4000, 2000)
- // to
- // select * from "scott".emp where sal > some (VALUES (4000), (2000))
- // The SqlNodeList become a RexSubQuery then optimized by
- // SubQueryRemoveRule.
- RelNode relNode =
- convertRowValues(
- bb, query, (SqlNodeList) query, false, targetRowType);
- final ImmutableList.Builder<RexNode> builder = ImmutableList.builder();
- for (SqlNode node : leftSqlKeys) {
- builder.add(bb.convertExpression(node));
- }
- final ImmutableList<RexNode> list = builder.build();
- assert relNode != null;
- subQuery.expr =
- createSubquery(subQuery.node.getKind(), relNode, list, call);
+ convertNodeListToSubQuery(
+ bb, subQuery, query, leftSqlKeys, targetRowType, call);
return;
}
return;
@@ -1339,7 +1336,12 @@ public class SqlToRelConverter {
//
// In such case, when converting SqlUpdate#condition, bb.root is null
// and it makes no sense to do the sub-query substitution.
+ // Instead, we convert to a RexSubQuery which can be optimized later.
if (bb.root == null) {
+ if (query instanceof SqlNodeList) {
+ convertNodeListToSubQuery(
+ bb, subQuery, query, leftSqlKeys, targetRowType, call);
+ }
return;
}
@@ -1921,6 +1923,48 @@ public class SqlToRelConverter {
return node;
}
+ /**
+ * Converts a {@link SqlNodeList} (for example an IN-list or VALUES list) into a relational
+ * expression and produces a Rex-level sub-query that references that relational expression.
+ *
+ * <p>For example, converts:
+ *
+ * <pre>{@code
+ * select * from "scott".emp where sal > some (4000, 2000)
+ * }</pre>
+ *
+ * to:
+ *
+ * <pre>{@code
+ * select * from "scott".emp where sal > some (VALUES (4000), (2000))
+ * }</pre>
+ *
+ * The SqlNodeList becomes a RexSubQuery then optimized by SubQueryRemoveRule.
+ *
+ * @param bb Blackboard containing the context
+ * @param subQuery The SubQuery to populate with the converted expression
+ * @param query The query node (expected to be a SqlNodeList)
+ * @param leftSqlKeys Left-hand key expressions for IN/SOME/ALL
+ * @param targetRowType The target row type for the conversion
+ * @param call The original SQL call
+ */
+ private void convertNodeListToSubQuery(
+ Blackboard bb,
+ SubQuery subQuery,
+ SqlNode query,
+ List<SqlNode> leftSqlKeys,
+ RelDataType targetRowType,
+ SqlCall call) {
+ RelNode relNode = convertRowValues(bb, query, (SqlNodeList) query, false, targetRowType);
+ final ImmutableList.Builder<RexNode> builder = ImmutableList.builder();
+ for (SqlNode node : leftSqlKeys) {
+ builder.add(bb.convertExpression(node));
+ }
+ final ImmutableList<RexNode> list = builder.build();
+ assert relNode != null;
+ subQuery.expr = createSubquery(subQuery.node.getKind(), relNode, list, call);
+ }
+
/**
* Gets the list size threshold under which {@link #convertInToOr} is used. Lists of this size
* or greater will instead be converted to use a join against an inline table ({@link
@@ -2670,15 +2714,15 @@ public class SqlToRelConverter {
convertCollectionTable(bb, call2);
return;
- // ----- FLINK MODIFICATION BEGIN -----
case LATERAL:
call = (SqlCall) from;
+
// Extract and analyze lateral part of join call.
assert call.getOperandList().size() == 1;
final SqlCall callLateral = call.operand(0);
convertFrom(bb, callLateral, fieldNames);
return;
- // ----- FLINK MODIFICATION END -----
+
default:
throw new AssertionError("not a join operator " + from);
}
@@ -3350,12 +3394,8 @@ public class SqlToRelConverter {
}
final ImmutableBitSet.Builder requiredColumns = ImmutableBitSet.builder();
final List<CorrelationId> correlNames = new ArrayList<>();
-
- // All correlations must refer the same namespace since correlation
- // produces exactly one correlation source.
- // The same source might be referenced by different variables since
- // DeferredLookups are not de-duplicated at create time.
- SqlValidatorNamespace prevNs = null;
+ // Mapping from (correlId, originalFieldIndex) to projectedFieldIndex for aggregation
+ final Map<Pair<CorrelationId, Integer>, Integer> fieldMapping = new HashMap<>();
for (CorrelationId correlName : correlatedVariables) {
DeferredLookup lookup =
@@ -3371,7 +3411,6 @@ public class SqlToRelConverter {
ImmutableList.of(lookup.originalRelName), nameMatcher, false, resolved);
assert resolved.count() == 1;
final SqlValidatorScope.Resolve resolve = resolved.only();
- final SqlValidatorNamespace foundNs = resolve.namespace;
final RelDataType rowType = resolve.rowType();
final int childNamespaceIndex = resolve.path.steps().get(0).i;
final SqlValidatorScope ancestorScope = resolve.scope;
@@ -3381,18 +3420,6 @@ public class SqlToRelConverter {
continue;
}
- if (prevNs == null) {
- prevNs = foundNs;
- } else {
- assert prevNs == foundNs
- : "All correlation variables should resolve"
- + " to the same namespace."
- + " Prev ns="
- + prevNs
- + ", new ns="
- + foundNs;
- }
-
int namespaceOffset = 0;
if (childNamespaceIndex > 0) {
// If not the first child, need to figure out the width
@@ -3410,9 +3437,9 @@ public class SqlToRelConverter {
while (topLevelFieldAccess.getReferenceExpr() instanceof RexFieldAccess) {
topLevelFieldAccess = (RexFieldAccess) topLevelFieldAccess.getReferenceExpr();
}
+ final int originalFieldIndex = topLevelFieldAccess.getField().getIndex();
final RelDataTypeField field =
- rowType.getFieldList()
- .get(topLevelFieldAccess.getField().getIndex() - namespaceOffset);
+ rowType.getFieldList().get(originalFieldIndex - namespaceOffset);
int pos = namespaceOffset + field.getIndex();
assert field.getType() == topLevelFieldAccess.getField().getType();
@@ -3427,6 +3454,7 @@ public class SqlToRelConverter {
// the root of the outer relation.
Integer projection = exprProjection.get(pos);
if (projection != null) {
+ fieldMapping.put(Pair.of(correlName, originalFieldIndex), projection);
pos = projection;
} else {
// correl not grouped
@@ -3458,6 +3486,19 @@ public class SqlToRelConverter {
// Add new node to leaves.
leaves.put(r, r.getRowType().getFieldCount());
}
+
+ // If there are field mappings (due to aggregation), rewrite the RelNode tree
+ // to update correlation variable row type and field indices
+ if (!fieldMapping.isEmpty()) {
+ r =
+ r.accept(
+ new CorrelationFieldMappingShuttle(
+ rexBuilder,
+ correlNames.get(0),
+ bb.root().getRowType(),
+ fieldMapping));
+ }
+
return new CorrelationUse(correlNames.get(0), requiredColumns.build(), r);
}
@@ -3532,7 +3573,7 @@ public class SqlToRelConverter {
final RelNode tempRightRel = requireNonNull(rightBlackboard.root, "rightBlackboard.root");
final JoinConditionType conditionType = join.getConditionType();
- final RexNode condition;
+ RexNode condition;
RelNode rightRel;
if (join.isNatural()) {
condition = convertNaturalCondition(getNamespace(left), getNamespace(right));
@@ -3561,6 +3602,8 @@ public class SqlToRelConverter {
throw Util.unexpected(conditionType);
}
}
+ condition = simplifyPredicate(condition);
+
final RelNode joinRel;
if (joinType == JoinType.ASOF || joinType == JoinType.LEFT_ASOF) {
SqlNode sqlMatchCondition =
@@ -3570,6 +3613,7 @@ public class SqlToRelConverter {
Pair<RexNode, RelNode> conditionAndRightNode =
convertOnCondition(fromBlackboard, sqlMatchCondition, leftRel, tempRightRel);
RexNode matchCondition = conditionAndRightNode.left;
+ matchCondition = simplifyPredicate(matchCondition);
rightRel = conditionAndRightNode.right;
joinRel =
createAsofJoin(
@@ -4666,14 +4710,27 @@ public class SqlToRelConverter {
targetColumnNameList.add(field.getName());
}
- RelNode sourceRel = convertSelect(sourceSelect, false);
+ // `sourceSelect` should contain target columns values plus source expressions
+ if (sourceSelect.getSelectList().size()
+ != targetTable.getRowType().getFieldCount()
+ + call.getSourceExpressionList().size()) {
+ throw new AssertionError(
+ "Unexpected select list size. Select list should contain both target table columns and "
+ + "set expressions");
+ }
+ RelNode sourceRel = convertSelect(sourceSelect, false);
bb.setRoot(sourceRel, false);
- ImmutableList.Builder<RexNode> rexNodeSourceExpressionListBuilder = ImmutableList.builder();
- for (SqlNode n : call.getSourceExpressionList()) {
- RexNode rn = bb.convertExpression(n);
- rexNodeSourceExpressionListBuilder.add(rn);
- }
+
+ // sourceRel already contains all source expressions. Only create references to those
+ // fields.
+ List<RexNode> rexExpressionList =
+ Util.transform(
+ Util.last(
+ sourceRel.getRowType().getFieldList(), targetColumnNameList.size()),
+ expressionField ->
+ new RexInputRef(
+ expressionField.getIndex(), expressionField.getType()));
return LogicalTableModify.create(
targetTable,
@@ -4681,7 +4738,7 @@ public class SqlToRelConverter {
sourceRel,
LogicalTableModify.Operation.UPDATE,
targetColumnNameList,
- rexNodeSourceExpressionListBuilder.build(),
+ rexExpressionList,
false);
}
@@ -5009,7 +5066,7 @@ public class SqlToRelConverter {
case ARRAY_QUERY_CONSTRUCTOR:
case MAP_QUERY_CONSTRUCTOR:
final RelRoot root = convertQuery(call.operand(0), false, true);
- input = root.rel;
+ input = root.project();
break;
default:
lastList.add(operand);
@@ -5315,6 +5372,7 @@ public class SqlToRelConverter {
mapping = null;
}
+ List<RelDataTypeField> fields = targetRowType.getFieldList();
for (SqlNode rowConstructor : values.getOperandList()) {
SqlCall newRowConst = (SqlCall) rowConstructor;
Blackboard tmpBb = createBlackboard(bb.scope, null, false);
@@ -5323,6 +5381,7 @@ public class SqlToRelConverter {
Ord.forEach(
newRowConst.getOperandList(),
(operand, i) -> {
+ RelDataType fieldType = fields.get(i).getType();
RexNode def;
if (processDefaults
&& operand.getKind() == SqlKind.DEFAULT
@@ -5336,6 +5395,9 @@ public class SqlToRelConverter {
} else {
def = tmpBb.convertExpression(operand);
}
+ if (!(def instanceof RexDynamicParam) && !def.getType().equals(fieldType)) {
+ def = rexBuilder.makeCast(operand.getParserPosition(), fieldType, def);
+ }
exps.add(def, SqlValidatorUtil.alias(operand, i));
});
@@ -5937,20 +5999,20 @@ public class SqlToRelConverter {
query = Iterables.getOnlyElement(call.getOperandList());
// let top=true to make the query be top-level query,
// then ORDER BY will be reserved.
- root = convertQueryRecursive(query, true, null);
- return RexSubQuery.array(root.rel);
+ root = convertQuery(query, false, true);
+ return RexSubQuery.array(root.project());
case MAP_QUERY_CONSTRUCTOR:
call = (SqlCall) expr;
query = Iterables.getOnlyElement(call.getOperandList());
- root = convertQueryRecursive(query, false, null);
- return RexSubQuery.map(root.rel);
+ root = convertQuery(query, false, true);
+ return RexSubQuery.map(root.project());
case MULTISET_QUERY_CONSTRUCTOR:
call = (SqlCall) expr;
query = Iterables.getOnlyElement(call.getOperandList());
- root = convertQueryRecursive(query, false, null);
- return RexSubQuery.multiset(root.rel);
+ root = convertQuery(query, false, true);
+ return RexSubQuery.multiset(root.project());
default:
break;
@@ -6157,14 +6219,14 @@ public class SqlToRelConverter {
}
@Override
- public int getGroupCount() {
+ public boolean hasEmptyGroup() {
if (agg != null) {
- return agg.groupExprs.size();
+ return SqlValidatorUtil.hasEmptyGroup(agg.groupExprs);
}
if (window != null) {
- return window.isAlwaysNonEmpty() ? 1 : 0;
+ return !window.isAlwaysNonEmpty();
}
- return -1;
+ return false;
}
@Override
@@ -6323,6 +6385,60 @@ public class SqlToRelConverter {
}
}
+ /**
+ * Shuttle that rewrites correlation field accesses to use projected field indices when
+ * correlation references aggregated relations.
+ */
+ private static class CorrelationFieldMappingShuttle extends RelHomogeneousShuttle {
+ private final RexBuilder rexBuilder;
+ private final CorrelationId targetCorrelId;
+ private final RelDataType newCorrelRowType;
+ private final Map<Pair<CorrelationId, Integer>, Integer> fieldMapping;
+
+ CorrelationFieldMappingShuttle(
+ RexBuilder rexBuilder,
+ CorrelationId targetCorrelId,
+ RelDataType newCorrelRowType,
+ Map<Pair<CorrelationId, Integer>, Integer> fieldMapping) {
+ this.rexBuilder = rexBuilder;
+ this.targetCorrelId = targetCorrelId;
+ this.newCorrelRowType = newCorrelRowType;
+ this.fieldMapping = fieldMapping;
+ }
+
+ @Override
+ public RelNode visit(RelNode other) {
+ return super.visit(other)
+ .accept(
+ new RexShuttle() {
+ @Override
+ public RexNode visitFieldAccess(RexFieldAccess fieldAccess) {
+ if (fieldAccess.getReferenceExpr()
+ instanceof RexCorrelVariable) {
+ RexCorrelVariable correlVar =
+ (RexCorrelVariable) fieldAccess.getReferenceExpr();
+ if (correlVar.id.equals(targetCorrelId)) {
+ Integer newIndex =
+ fieldMapping.get(
+ Pair.of(
+ correlVar.id,
+ fieldAccess
+ .getField()
+ .getIndex()));
+ if (newIndex != null) {
+ return rexBuilder.makeFieldAccess(
+ rexBuilder.makeCorrel(
+ newCorrelRowType, correlVar.id),
+ newIndex);
+ }
+ }
+ }
+ return super.visitFieldAccess(fieldAccess);
+ }
+ });
+ }
+ }
+
/** A default implementation of SubQueryConverter that does no conversion. */
private static class NoOpSubQueryConverter implements SubQueryConverter {
@Override
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecMatch.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecMatch.java
index 8da68878070..f662b87e606 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecMatch.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecMatch.java
@@ -68,6 +68,7 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonPro
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexNodeAndFieldIndex;
import org.apache.calcite.sql.SqlMatchRecognize;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
@@ -367,6 +368,12 @@ public abstract class CommonExecMatch extends ExecNodeBase<RowData>
}
}
+ @Override
+ public Pattern<RowData, RowData> visitNodeAndFieldIndex(
+ RexNodeAndFieldIndex nodeAndFieldIndex) {
+ throw new UnsupportedOperationException("not supported yet");
+ }
+
@Override
public Pattern<RowData, RowData> visitNode(RexNode rexNode) {
throw new TableException(
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/LogicalCorrelateToJoinFromTemporalTableFunctionRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/LogicalCorrelateToJoinFromTemporalTableFunctionRule.java
index d80062919e5..749d8a3353b 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/LogicalCorrelateToJoinFromTemporalTableFunctionRule.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/LogicalCorrelateToJoinFromTemporalTableFunctionRule.java
@@ -49,6 +49,7 @@ import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexFieldAccess;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexNodeAndFieldIndex;
import org.apache.calcite.rex.RexVisitorImpl;
import org.apache.calcite.sql.SqlOperator;
import org.immutables.value.Value;
@@ -336,6 +337,11 @@ class CorrelatedFieldAccessRemoval extends RexDefaultVisitor<RexNode> {
return rexBuilder.makeInputRef(leftSide, leftIndex);
}
+ @Override
+ public RexNode visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex) {
+ throw new UnsupportedOperationException("not supported yet");
+ }
+
@Override
public RexNode visitInputRef(RexInputRef inputRef) {
return inputRef;
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/RemoteCorrelateSplitRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/RemoteCorrelateSplitRule.java
index 1515b0975ff..1bf31d8cb76 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/RemoteCorrelateSplitRule.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/RemoteCorrelateSplitRule.java
@@ -37,6 +37,7 @@ import org.apache.calcite.rex.RexCorrelVariable;
import org.apache.calcite.rex.RexFieldAccess;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexNodeAndFieldIndex;
import org.apache.calcite.rex.RexProgram;
import org.apache.calcite.rex.RexProgramBuilder;
import org.apache.calcite.rex.RexUtil;
@@ -131,6 +132,11 @@ public class RemoteCorrelateSplitRule extends RelRule<RemoteCorrelateSplitRule.C
}
}
+ @Override
+ public RexNode visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex) {
+ throw new UnsupportedOperationException("not supported yet");
+ }
+
@Override
public RexNode visitCall(RexCall call) {
List<RexNode> newProjects =
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/SplitPythonConditionFromCorrelateRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/SplitPythonConditionFromCorrelateRule.java
index f2846447176..7677bf371dc 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/SplitPythonConditionFromCorrelateRule.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/SplitPythonConditionFromCorrelateRule.java
@@ -34,6 +34,7 @@ import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexNodeAndFieldIndex;
import org.apache.calcite.rex.RexProgram;
import org.apache.calcite.rex.RexProgramBuilder;
import org.apache.calcite.rex.RexUtil;
@@ -217,6 +218,11 @@ public class SplitPythonConditionFromCorrelateRule
.collect(Collectors.toList()));
}
+ @Override
+ public RexNode visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex) {
+ throw new UnsupportedOperationException("not supported yet");
+ }
+
@Override
public RexNode visitNode(RexNode rexNode) {
return rexNode;
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/common/CommonPhysicalMatchRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/common/CommonPhysicalMatchRule.java
index 5476fbbbebe..a4d6e04c1cf 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/common/CommonPhysicalMatchRule.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/common/CommonPhysicalMatchRule.java
@@ -37,6 +37,7 @@ import org.apache.calcite.rel.convert.ConverterRule;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexNodeAndFieldIndex;
import org.apache.calcite.sql.SqlAggFunction;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.util.ImmutableBitSet;
@@ -169,6 +170,11 @@ public abstract class CommonPhysicalMatchRule extends ConverterRule {
return null;
}
+ @Override
+ public Object visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex) {
+ throw new UnsupportedOperationException("not supported yet");
+ }
+
@Override
public Object visitNode(RexNode rexNode) {
return null;
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/AsyncUtil.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/AsyncUtil.java
index e283252b704..b435c2d8a30 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/AsyncUtil.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/AsyncUtil.java
@@ -26,6 +26,7 @@ import org.apache.flink.util.Preconditions;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexNodeAndFieldIndex;
/** Utility class for working with async function calls in RexNodes. */
public class AsyncUtil {
@@ -135,6 +136,11 @@ public class AsyncUtil {
|| (recursive
&& call.getOperands().stream().anyMatch(node -> node.accept(this)));
}
+
+ @Override
+ public Boolean visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex) {
+ throw new UnsupportedOperationException("not supported yet");
+ }
}
/**
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/ConstantFoldingUtil.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/ConstantFoldingUtil.java
index c8e3e40d7a3..3803fa9b2f4 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/ConstantFoldingUtil.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/ConstantFoldingUtil.java
@@ -23,6 +23,7 @@ import org.apache.flink.table.planner.utils.ShortcutUtils;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexNodeAndFieldIndex;
/** Utility for deciding whether than expression supports constant folding or not. */
public class ConstantFoldingUtil {
@@ -55,5 +56,10 @@ public class ConstantFoldingUtil {
return supportsConstantFolding
&& (call.getOperands().stream().allMatch(node -> node.accept(this)));
}
+
+ @Override
+ public Boolean visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex) {
+ throw new UnsupportedOperationException("not supported yet");
+ }
}
}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/PythonUtil.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/PythonUtil.java
index 8691b30cd0d..360129870cf 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/PythonUtil.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/PythonUtil.java
@@ -40,6 +40,7 @@ import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexFieldAccess;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexNodeAndFieldIndex;
import org.apache.calcite.sql.SqlAggFunction;
import org.apache.calcite.sql.SqlKind;
@@ -275,6 +276,11 @@ public class PythonUtil {
return fieldAccess.getReferenceExpr().accept(this);
}
+ @Override
+ public Boolean visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex) {
+ throw new UnsupportedOperationException("not supported yet");
+ }
+
@Override
public Boolean visitNode(RexNode rexNode) {
return false;
@@ -307,6 +313,11 @@ public class PythonUtil {
}
}
+ @Override
+ public Boolean visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex) {
+ throw new UnsupportedOperationException("not supported yet");
+ }
+
@Override
public Boolean visitCall(RexCall call) {
if (call.getKind() == SqlKind.AS) {
diff --git a/flink-table/flink-table-planner/src/main/resources/META-INF/NOTICE b/flink-table/flink-table-planner/src/main/resources/META-INF/NOTICE
index 0688c621bf0..943be5e3d5f 100644
--- a/flink-table/flink-table-planner/src/main/resources/META-INF/NOTICE
+++ b/flink-table/flink-table-planner/src/main/resources/META-INF/NOTICE
@@ -8,13 +8,13 @@ This project bundles the following dependencies under the Apache Software Licens
- com.google.guava:guava:33.4.0-jre
- com.google.guava:failureaccess:1.0.2
-- org.apache.calcite:calcite-core:1.40.0
-- org.apache.calcite:calcite-linq4j:1.40.0
-- org.apache.calcite.avatica:avatica-core:1.26.0
+- org.apache.calcite:calcite-core:1.41.0
+- org.apache.calcite:calcite-linq4j:1.41.0
+- org.apache.calcite.avatica:avatica-core:1.27.0
- org.apache.commons:commons-math3:3.6.1
-- org.apache.commons:commons-text:1.10.0
- commons-codec:commons-codec:1.15
- commons-io:commons-io:2.15.1
+- org.jooq:joou-java-6:0.9.4
This project bundles the following dependencies under the MIT License. (http://www.opensource.org/licenses/mit-license.php)
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ExprCodeGenerator.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ExprCodeGenerator.scala
index b05eac28261..9427dace35d 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ExprCodeGenerator.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ExprCodeGenerator.scala
@@ -1026,4 +1026,9 @@ class ExprCodeGenerator(
ShortcutUtils.isDeterministicThroughProgram(
node,
CodeGenUtils.getExprsFromProgramOrNull(rexProgram))
+
+ override def visitNodeAndFieldIndex(
+ nodeAndFieldIndex: RexNodeAndFieldIndex): GeneratedExpression = {
+ throw new CodeGenException("RexNodeAndFieldIndex are not supported yet.")
+ }
}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcSplitRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcSplitRule.scala
index f4537225dbc..b00aaa491c8 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcSplitRule.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcSplitRule.scala
@@ -18,12 +18,13 @@
package org.apache.flink.table.planner.plan.rules.logical
import org.apache.flink.table.functions.ScalarFunction
+import org.apache.flink.table.planner.codegen.CodeGenException
import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCalc
import org.apache.flink.table.planner.plan.utils.{InputRefVisitor, RexDefaultVisitor}
import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall}
import org.apache.calcite.plan.RelOptRule.{any, operand}
-import org.apache.calcite.rex.{RexBuilder, RexCall, RexCorrelVariable, RexFieldAccess, RexInputRef, RexLocalRef, RexNode, RexProgram}
+import org.apache.calcite.rex.{RexBuilder, RexCall, RexCorrelVariable, RexFieldAccess, RexInputRef, RexLocalRef, RexNode, RexNodeAndFieldIndex, RexProgram}
import org.apache.calcite.sql.validate.SqlValidatorUtil
import java.util.function.Function
@@ -497,6 +498,10 @@ class ScalarFunctionSplitter(
new RexInputRef(fieldsRexCall(rexCallIndex), remoteCall.getType),
node.getField.getIndex)
}
+
+ override def visitNodeAndFieldIndex(nodeAndFieldIndex: RexNodeAndFieldIndex): RexNode = {
+ throw new CodeGenException("RexNodeAndFieldIndex are not supported yet.")
+ }
}
/**
@@ -545,4 +550,8 @@ private class ExtractedFunctionInputRewriter(
}
override def visitNode(rexNode: RexNode): RexNode = rexNode
+
+ override def visitNodeAndFieldIndex(nodeAndFieldIndex: RexNodeAndFieldIndex): RexNode = {
+ throw new CodeGenException("RexNodeAndFieldIndex are not supported yet.")
+ }
}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/MatchUtil.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/MatchUtil.scala
index 0c1760e8510..6486114115d 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/MatchUtil.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/MatchUtil.scala
@@ -18,13 +18,14 @@
package org.apache.flink.table.planner.plan.utils
import org.apache.flink.table.api.ValidationException
+import org.apache.flink.table.planner.codegen.CodeGenException
import org.apache.flink.table.planner.codegen.MatchCodeGenerator.ALL_PATTERN_VARIABLE
import org.apache.flink.table.planner.functions.sql.FlinkSqlOperatorTable
import org.apache.flink.table.planner.plan.logical.MatchRecognize
import org.apache.flink.table.planner.plan.nodes.exec.spec.{MatchSpec, PartitionSpec}
import _root_.scala.collection.JavaConversions._
-import org.apache.calcite.rex.{RexCall, RexNode, RexPatternFieldRef}
+import org.apache.calcite.rex.{RexCall, RexNode, RexNodeAndFieldIndex, RexPatternFieldRef}
import org.apache.calcite.sql.fun.SqlStdOperatorTable
object MatchUtil {
@@ -57,6 +58,10 @@ object MatchUtil {
}
override def visitNode(rexNode: RexNode): Option[String] = None
+
+ override def visitNodeAndFieldIndex(nodeAndFieldIndex: RexNodeAndFieldIndex): Option[String] = {
+ throw new CodeGenException("RexNodeAndFieldIndex are not supported yet.")
+ }
}
/** Convert [[MatchRecognize]] to [[MatchSpec]]. */
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/RexNodeExtractor.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/RexNodeExtractor.scala
index 59c1ae9e2e3..8ebefbbdc95 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/RexNodeExtractor.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/RexNodeExtractor.scala
@@ -27,6 +27,7 @@ import org.apache.flink.table.expressions.ApiExpressionUtils._
import org.apache.flink.table.functions.{BuiltInFunctionDefinition, FunctionIdentifier}
import org.apache.flink.table.functions.BuiltInFunctionDefinitions.{AND, CAST, OR, TRY_CAST}
import org.apache.flink.table.planner.calcite.FlinkTypeFactory
+import org.apache.flink.table.planner.codegen.CodeGenException
import org.apache.flink.table.planner.functions.sql.FlinkSqlOperatorTable
import org.apache.flink.table.planner.utils.Logging
import org.apache.flink.table.planner.utils.TimestampStringUtils.toLocalDateTime
@@ -627,4 +628,8 @@ class RexNodeToExpressionConverter(
str.replaceAll("\\s|_", "")
}
+ override def visitNodeAndFieldIndex(
+ nodeAndFieldIndex: RexNodeAndFieldIndex): Option[ResolvedExpression] = {
+ throw new CodeGenException("RexNodeAndFieldIndex are not supported yet.")
+ }
}
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/ValuesTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/ValuesTest.xml
index 089c876dea6..645ff6b5205 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/ValuesTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/ValuesTest.xml
@@ -25,9 +25,9 @@ limitations under the License.
LogicalProject(a=[$0], b=[$1]), rowType=[RecordType(INTEGER a, DECIMAL(20, 1) b)]
+- LogicalProject(a=[$0], b=[$1]), rowType=[RecordType(INTEGER a, DECIMAL(20, 1) b)]
+- LogicalUnion(all=[true]), rowType=[RecordType(INTEGER EXPR$0, DECIMAL(20, 1) EXPR$1)]
- :- LogicalProject(EXPR$0=[1], EXPR$1=[2.0:DECIMAL(2, 1)]), rowType=[RecordType(INTEGER EXPR$0, DECIMAL(2, 1) EXPR$1)]
+ :- LogicalProject(EXPR$0=[1], EXPR$1=[2.0:DECIMAL(20, 1)]), rowType=[RecordType(INTEGER EXPR$0, DECIMAL(20, 1) EXPR$1)]
: +- LogicalValues(tuples=[[{ 0 }]]), rowType=[RecordType(INTEGER ZERO)]
- +- LogicalProject(EXPR$0=[3], EXPR$1=[4:BIGINT]), rowType=[RecordType(INTEGER EXPR$0, BIGINT EXPR$1)]
+ +- LogicalProject(EXPR$0=[3], EXPR$1=[4:DECIMAL(20, 1)]), rowType=[RecordType(INTEGER EXPR$0, DECIMAL(20, 1) EXPR$1)]
+- LogicalValues(tuples=[[{ 0 }]]), rowType=[RecordType(INTEGER ZERO)]
]]>
</Resource>
@@ -36,7 +36,7 @@ LogicalProject(a=[$0], b=[$1]), rowType=[RecordType(INTEGER a, DECIMAL(20, 1) b)
Union(all=[true], union=[EXPR$0, EXPR$1]), rowType=[RecordType(INTEGER EXPR$0, DECIMAL(20, 1) EXPR$1)]
:- Calc(select=[1 AS EXPR$0, 2.0 AS EXPR$1]), rowType=[RecordType(INTEGER EXPR$0, DECIMAL(20, 1) EXPR$1)]
: +- Values(tuples=[[{ 0 }]], values=[ZERO]), rowType=[RecordType(INTEGER ZERO)]
-+- Calc(select=[3 AS EXPR$0, 4.0 AS EXPR$1]), rowType=[RecordType(INTEGER EXPR$0, DECIMAL(20, 1) EXPR$1)]
++- Calc(select=[3 AS EXPR$0, 4 AS EXPR$1]), rowType=[RecordType(INTEGER EXPR$0, DECIMAL(20, 1) EXPR$1)]
+- Values(tuples=[[{ 0 }]], values=[ZERO]), rowType=[RecordType(INTEGER ZERO)]
]]>
</Resource>
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/agg/GroupingSetsTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/agg/GroupingSetsTest.xml
index ae9e1459285..af6bc3c4d93 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/agg/GroupingSetsTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/agg/GroupingSetsTest.xml
@@ -182,10 +182,9 @@ LogicalUnion(all=[true])
: +- LogicalAggregate(group=[{0}], groups=[[{0}, {}]], c=[COUNT()])
: +- LogicalProject(deptno=[$7])
: +- LogicalTableScan(table=[[default_catalog, default_database, scott_emp]])
-+- LogicalProject(deptno=[$0], g=[1:BIGINT], c=[$1])
- +- LogicalAggregate(group=[{0}], groups=[[{}]], c=[COUNT()])
- +- LogicalProject(deptno=[$7])
- +- LogicalTableScan(table=[[default_catalog, default_database, scott_emp]])
++- LogicalProject(deptno=[null:INTEGER], g=[1:BIGINT], c=[$0])
+ +- LogicalAggregate(group=[{}], c=[COUNT()])
+ +- LogicalTableScan(table=[[default_catalog, default_database, scott_emp]])
]]>
</Resource>
<Resource name="optimized exec plan">
@@ -196,13 +195,14 @@ Union(all=[true], union=[deptno, g, c])
: +- Exchange(distribution=[hash[deptno, $e]])
: +- LocalHashAggregate(groupBy=[deptno, $e], select=[deptno, $e, Partial_COUNT(*) AS count1$0])
: +- Expand(projects=[{deptno, 0 AS $e}, {null AS deptno, 1 AS $e}])
-: +- Calc(select=[deptno])(reuse_id=[1])
-: +- TableSourceScan(table=[[default_catalog, default_database, scott_emp]], fields=[empno, ename, job, mgr, hiredate, sal, comm, deptno])
-+- Calc(select=[deptno, 1 AS g, c])
- +- HashAggregate(isMerge=[true], groupBy=[deptno], select=[deptno, Final_COUNT(count1$0) AS c])
- +- Exchange(distribution=[hash[deptno]])
- +- LocalHashAggregate(groupBy=[deptno], select=[deptno, Partial_COUNT(*) AS count1$0])
- +- Reused(reference_id=[1])
+: +- Calc(select=[deptno])
+: +- TableSourceScan(table=[[default_catalog, default_database, scott_emp]], fields=[empno, ename, job, mgr, hiredate, sal, comm, deptno])(reuse_id=[1])
++- Calc(select=[null:INTEGER AS deptno, 1 AS g, c])
+ +- HashAggregate(isMerge=[true], select=[Final_COUNT(count1$0) AS c])
+ +- Exchange(distribution=[single])
+ +- LocalHashAggregate(select=[Partial_COUNT(*) AS count1$0])
+ +- Calc(select=[0 AS $f0])
+ +- Reused(reference_id=[1])
]]>
</Resource>
</TestCase>
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashJoinTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashJoinTest.xml
index dc0d9c3e721..1a38b9d05e9 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashJoinTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashJoinTest.xml
@@ -472,7 +472,7 @@ Calc(select=[d, e, f])
<Resource name="ast">
<![CDATA[
LogicalProject(d=[$3], e=[$4], f=[$5])
-+- LogicalFilter(condition=[=($3, null)])
++- LogicalFilter(condition=[false])
+- LogicalJoin(condition=[=($0, $3)], joinType=[left])
:- LogicalTableScan(table=[[default_catalog, default_database, MyTable1]])
+- LogicalTableScan(table=[[default_catalog, default_database, MyTable2]])
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/NestedLoopJoinTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/NestedLoopJoinTest.xml
index 4f6c8e1e9a4..8ce9c14a314 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/NestedLoopJoinTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/NestedLoopJoinTest.xml
@@ -788,7 +788,7 @@ Calc(select=[d, e, f])
<Resource name="ast">
<![CDATA[
LogicalProject(d=[$3], e=[$4], f=[$5])
-+- LogicalFilter(condition=[=($3, null)])
++- LogicalFilter(condition=[false])
+- LogicalJoin(condition=[=($0, $3)], joinType=[left])
:- LogicalTableScan(table=[[default_catalog, default_database, MyTable1]])
+- LogicalTableScan(table=[[default_catalog, default_database, MyTable2]])
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashJoinTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashJoinTest.xml
index 1a1470c7146..c6e835b02ed 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashJoinTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashJoinTest.xml
@@ -586,7 +586,7 @@ Calc(select=[d, e, f])
<Resource name="ast">
<![CDATA[
LogicalProject(d=[$3], e=[$4], f=[$5])
-+- LogicalFilter(condition=[=($3, null)])
++- LogicalFilter(condition=[false])
+- LogicalJoin(condition=[=($0, $3)], joinType=[left])
:- LogicalTableScan(table=[[default_catalog, default_database, MyTable1]])
+- LogicalTableScan(table=[[default_catalog, default_database, MyTable2]])
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeJoinTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeJoinTest.xml
index b125f435441..44ad033e09e 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeJoinTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeJoinTest.xml
@@ -586,7 +586,7 @@ Calc(select=[d, e, f])
<Resource name="ast">
<![CDATA[
LogicalProject(d=[$3], e=[$4], f=[$5])
-+- LogicalFilter(condition=[=($3, null)])
++- LogicalFilter(condition=[false])
+- LogicalJoin(condition=[=($0, $3)], joinType=[left])
:- LogicalTableScan(table=[[default_catalog, default_database, MyTable1]])
+- LogicalTableScan(table=[[default_catalog, default_database, MyTable2]])
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/DecomposeGroupingSetsRuleTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/DecomposeGroupingSetsRuleTest.xml
index b7763815535..5c8620535ae 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/DecomposeGroupingSetsRuleTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/DecomposeGroupingSetsRuleTest.xml
@@ -29,10 +29,9 @@ LogicalUnion(all=[true])
: +- LogicalAggregate(group=[{0}], groups=[[{0}, {}]], c=[COUNT()])
: +- LogicalProject(a=[$0])
: +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
-+- LogicalProject(a=[$0], g=[1:BIGINT], c=[$1])
- +- LogicalAggregate(group=[{0}], groups=[[{}]], c=[COUNT()])
- +- LogicalProject(a=[$0])
- +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
++- LogicalProject(a=[null:INTEGER], g=[1:BIGINT], c=[$0])
+ +- LogicalAggregate(group=[{}], c=[COUNT()])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
]]>
</Resource>
<Resource name="optimized rel plan">
@@ -43,9 +42,9 @@ FlinkLogicalUnion(all=[true])
: +- FlinkLogicalExpand(projects=[{a, 0 AS $e}, {null AS a, 1 AS $e}])
: +- FlinkLogicalCalc(select=[a])
: +- FlinkLogicalTableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c])
-+- FlinkLogicalCalc(select=[a, 1 AS g, c])
- +- FlinkLogicalAggregate(group=[{0}], groups=[[{}]], c=[COUNT()])
- +- FlinkLogicalCalc(select=[a])
++- FlinkLogicalCalc(select=[null:INTEGER AS a, 1 AS g, c])
+ +- FlinkLogicalAggregate(group=[{}], c=[COUNT()])
+ +- FlinkLogicalCalc(select=[0 AS $f0])
+- FlinkLogicalTableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c])
]]>
</Resource>
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRuleTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRuleTest.xml
index 9d2ed41f764..3850d355889 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRuleTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRuleTest.xml
@@ -563,7 +563,7 @@ LogicalProject(a1=[$0], b1=[$1], c1=[$2], b2=[$3], c2=[$4], a2=[$5])
<Resource name="ast">
<![CDATA[
LogicalProject(a1=[$0], b1=[$1], c1=[$2], b2=[$3], c2=[$4], a2=[$5])
-+- LogicalFilter(condition=[=($5, null)])
++- LogicalFilter(condition=[false])
+- LogicalJoin(condition=[=($0, $5)], joinType=[inner])
:- LogicalTableScan(table=[[default_catalog, default_database, MyTable1]])
+- LogicalTableScan(table=[[default_catalog, default_database, MyTable2]])
@@ -1054,7 +1054,7 @@ LogicalProject(a1=[$0], b1=[$1], c1=[$2], b2=[$3], c2=[$4], a2=[$5])
<Resource name="ast">
<![CDATA[
LogicalProject(a1=[$0], b1=[$1], c1=[$2], b2=[$3], c2=[$4], a2=[$5])
-+- LogicalFilter(condition=[=($5, null)])
++- LogicalFilter(condition=[false])
+- LogicalJoin(condition=[=($0, $5)], joinType=[left])
:- LogicalTableScan(table=[[default_catalog, default_database, MyTable1]])
+- LogicalTableScan(table=[[default_catalog, default_database, MyTable2]])
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/RemoveSingleAggregateRuleTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/RemoveSingleAggregateRuleTest.xml
index 11e9a158b07..31c238f0afb 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/RemoveSingleAggregateRuleTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/RemoveSingleAggregateRuleTest.xml
@@ -35,15 +35,21 @@ LogicalProject(EXPR$0=[-($0, 1)])
<Resource name="optimized rel plan">
<![CDATA[
Calc(select=[-(CASE(IS NULL($f1), 0, $f1), 1) AS EXPR$0])
-+- HashJoin(joinType=[LeftOuterJoin], where=[=(i, y)], select=[i, y, $f1], build=[right])
++- HashJoin(joinType=[LeftOuterJoin], where=[IS NOT DISTINCT FROM(i, y)], select=[i, y, $f1], build=[right])
:- Exchange(distribution=[hash[i]])
: +- Calc(select=[i])
: +- TableSourceScan(table=[[default_catalog, default_database, bar]], fields=[i, s])
- +- HashAggregate(isMerge=[true], groupBy=[y], select=[y, Final_COUNT(count$0) AS $f1])
- +- Exchange(distribution=[hash[y]])
- +- LocalHashAggregate(groupBy=[y], select=[y, Partial_COUNT(x) AS count$0])
- +- Calc(select=[x, y], where=[IS NOT NULL(y)])
- +- TableSourceScan(table=[[default_catalog, default_database, foo]], fields=[x, y])
+ +- Calc(select=[i AS y, CASE(IS NOT NULL($f1), $f1, 0) AS $f1])
+ +- HashJoin(joinType=[LeftOuterJoin], where=[IS NOT DISTINCT FROM(i, y)], select=[i, y, $f1], build=[left])
+ :- HashAggregate(isMerge=[true], groupBy=[i], select=[i])
+ : +- Exchange(distribution=[hash[i]])
+ : +- LocalHashAggregate(groupBy=[i], select=[i])
+ : +- TableSourceScan(table=[[default_catalog, default_database, bar]], fields=[i, s])
+ +- HashAggregate(isMerge=[true], groupBy=[y], select=[y, Final_COUNT(count$0) AS $f1])
+ +- Exchange(distribution=[hash[y]])
+ +- LocalHashAggregate(groupBy=[y], select=[y, Partial_COUNT(x) AS count$0])
+ +- Calc(select=[x, y], where=[IS NOT NULL(y)])
+ +- TableSourceScan(table=[[default_catalog, default_database, foo]], fields=[x, y])
]]>
</Resource>
</TestCase>
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/SimplifyFilterConditionRuleTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/SimplifyFilterConditionRuleTest.xml
index c32350b43b6..466f054085f 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/SimplifyFilterConditionRuleTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/SimplifyFilterConditionRuleTest.xml
@@ -179,10 +179,10 @@ LogicalProject(a=[$0], b=[$1], c=[$2])
<Resource name="ast">
<![CDATA[
LogicalProject(a=[$0], b=[$1], c=[$2])
-+- LogicalFilter(condition=[AND(EXISTS({
++- LogicalFilter(condition=[EXISTS({
LogicalFilter(condition=[OR(AND(=($0, 1), =($1, 2)), AND(NOT(<>($0, 1)), =($1, 3)))])
LogicalTableScan(table=[[default_catalog, default_database, y]])
-}), true)])
+})])
+- LogicalTableScan(table=[[default_catalog, default_database, x]])
]]>
</Resource>
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/SimplifyJoinConditionRuleTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/SimplifyJoinConditionRuleTest.xml
index ec5e4f7f151..1ec0eaf18bb 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/SimplifyJoinConditionRuleTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/SimplifyJoinConditionRuleTest.xml
@@ -59,15 +59,19 @@ LogicalAggregate(group=[{}], EXPR$0=[COUNT()])
<![CDATA[
LogicalProject(a=[$0])
+- LogicalFilter(condition=[=($1, CASE(IS NULL($5), 0:BIGINT, $5))])
- +- LogicalJoin(condition=[AND(=($0, $3), IS NOT DISTINCT FROM($1, $4))], joinType=[left])
+ +- LogicalJoin(condition=[AND(IS NOT DISTINCT FROM($0, $3), IS NOT DISTINCT FROM($1, $4))], joinType=[left])
:- LogicalTableScan(table=[[default_catalog, default_database, MyTable1]])
- +- LogicalAggregate(group=[{0, 1}], EXPR$0=[COUNT()])
- +- LogicalProject(a=[$3], b=[$4])
- +- LogicalJoin(condition=[AND(=($0, $3), OR(<($0, 2), =($4, 5)))], joinType=[inner])
- :- LogicalTableScan(table=[[default_catalog, default_database, MyTable2]])
- +- LogicalAggregate(group=[{0, 1}])
- +- LogicalProject(a=[$0], b=[$1])
- +- LogicalTableScan(table=[[default_catalog, default_database, MyTable1]])
+ +- LogicalProject(a=[$0], b=[$1], EXPR$0=[CASE(IS NOT NULL($4), $4, 0)])
+ +- LogicalJoin(condition=[AND(IS NOT DISTINCT FROM($0, $2), IS NOT DISTINCT FROM($1, $3))], joinType=[left])
+ :- LogicalAggregate(group=[{0, 1}])
+ : +- LogicalTableScan(table=[[default_catalog, default_database, MyTable1]])
+ +- LogicalAggregate(group=[{0, 1}], EXPR$0=[COUNT()])
+ +- LogicalProject(a=[$3], b=[$4])
+ +- LogicalJoin(condition=[AND(=($0, $3), OR(<($0, 2), =($4, 5)))], joinType=[inner])
+ :- LogicalTableScan(table=[[default_catalog, default_database, MyTable2]])
+ +- LogicalAggregate(group=[{0, 1}])
+ +- LogicalProject(a=[$0], b=[$1])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable1]])
]]>
</Resource>
</TestCase>
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQueryAntiJoinTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQueryAntiJoinTest.xml
index 0e9221a7d85..5ee0b2fc55d 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQueryAntiJoinTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQueryAntiJoinTest.xml
@@ -96,33 +96,41 @@ LogicalProject(c=[$2])
+- LogicalFilter(condition=[OR(=($3, 0), AND(IS NULL($13), >=($4, $3)))])
+- LogicalJoin(condition=[=(CASE(AND(IS NOT TRUE(OR(IS NULL($1), =($6, 0))), IS NOT NULL($10)), 1, 2), $11)], joinType=[left])
:- LogicalJoin(condition=[AND(=($0, $9), =($1, $8))], joinType=[left])
- : :- LogicalJoin(condition=[=($0, $5)], joinType=[left])
+ : :- LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $5)], joinType=[left])
: : :- LogicalJoin(condition=[true], joinType=[inner])
: : : :- LogicalTableScan(table=[[default_catalog, default_database, l]])
- : : : +- LogicalAggregate(group=[{}], c=[COUNT()], ck=[COUNT($0, $1)])
- : : : +- LogicalProject(d=[$0], e=[$1])
+ : : : +- LogicalAggregate(group=[{}], c=[COUNT()], ck=[COUNT() FILTER $0])
+ : : : +- LogicalProject($f2=[OR(IS NOT NULL($0), IS NOT NULL($1))])
: : : +- LogicalTableScan(table=[[default_catalog, default_database, r]])
- : : +- LogicalAggregate(group=[{0}], c=[COUNT()], ck=[COUNT($1)])
- : : +- LogicalProject(i=[$0], j=[$1])
- : : +- LogicalFilter(condition=[IS NOT NULL($0)])
- : : +- LogicalTableScan(table=[[default_catalog, default_database, t]])
+ : : +- LogicalProject(i=[$0], c=[CASE(IS NOT NULL($2), $2, 0)], ck=[CASE(IS NOT NULL($3), $3, 0)])
+ : : +- LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])
+ : : :- LogicalAggregate(group=[{0}])
+ : : : +- LogicalJoin(condition=[true], joinType=[inner])
+ : : : :- LogicalTableScan(table=[[default_catalog, default_database, l]])
+ : : : +- LogicalAggregate(group=[{}], c=[COUNT()], ck=[COUNT() FILTER $0])
+ : : : +- LogicalProject($f2=[OR(IS NOT NULL($0), IS NOT NULL($1))])
+ : : : +- LogicalTableScan(table=[[default_catalog, default_database, r]])
+ : : +- LogicalAggregate(group=[{0}], c=[COUNT()], ck=[COUNT($1)])
+ : : +- LogicalProject(i=[$0], j=[$1])
+ : : +- LogicalFilter(condition=[IS NOT NULL($0)])
+ : : +- LogicalTableScan(table=[[default_catalog, default_database, t]])
: +- LogicalFilter(condition=[IS NOT NULL($0)])
: +- LogicalProject(j=[$0], i2=[$1], $f2=[true])
: +- LogicalAggregate(group=[{0, 1}])
: +- LogicalProject(j=[$1], i2=[$0])
: +- LogicalFilter(condition=[IS NOT NULL($0)])
: +- LogicalTableScan(table=[[default_catalog, default_database, t]])
- +- LogicalJoin(condition=[true], joinType=[left])
- :- LogicalJoin(condition=[true], joinType=[left])
- : :- LogicalFilter(condition=[=(4, $1)])
- : : +- LogicalProject(d=[$0], e=[$1], $f2=[true])
- : : +- LogicalAggregate(group=[{0, 1}])
- : : +- LogicalProject(d=[$0], e=[$1])
- : : +- LogicalTableScan(table=[[default_catalog, default_database, r]])
- : +- LogicalProject($f0=[true], dummy=[$0])
- : +- LogicalValues(tuples=[[{ true }]])
- +- LogicalProject($f0=[true], dummy=[$0])
- +- LogicalValues(tuples=[[{ true }]])
+ +- LogicalFilter(condition=[=(CASE(IS NULL($3), 3, IS NOT NULL($5), 4, 5), $1)])
+ +- LogicalJoin(condition=[true], joinType=[left])
+ :- LogicalJoin(condition=[true], joinType=[left])
+ : :- LogicalProject(d=[$0], e=[$1], $f2=[true])
+ : : +- LogicalAggregate(group=[{0, 1}])
+ : : +- LogicalProject(d=[$0], e=[$1])
+ : : +- LogicalTableScan(table=[[default_catalog, default_database, r]])
+ : +- LogicalProject($f0=[true], dummy=[$0])
+ : +- LogicalValues(tuples=[[{ true }]])
+ +- LogicalProject($f0=[true], dummy=[$0])
+ +- LogicalValues(tuples=[[{ true }]])
]]>
</Resource>
</TestCase>
@@ -901,15 +909,21 @@ LogicalProject(d=[$0], e=[$1])
<Resource name="optimized rel plan">
<![CDATA[
LogicalProject(c=[$2])
-+- LogicalFilter(condition=[OR(=($3, 0), AND(IS NULL($11), >=($4, $3)))])
- +- LogicalJoin(condition=[=(CASE(OR(=($5, 0), AND(IS NULL($8), >=($6, $5), IS NOT NULL($1))), 3, 4), $10)], joinType=[left])
- :- LogicalJoin(condition=[=($1, $7)], joinType=[left])
++- LogicalFilter(condition=[OR(=($3, 0), AND(IS NULL($13), >=($4, $3)))])
+ +- LogicalJoin(condition=[AND(=(CASE(IS NULL($6), 1, 2), $11), =(CASE(OR(=($7, 0), AND(IS NULL($10), >=($8, $7), IS NOT NULL($1))), 3, 4), $12))], joinType=[left])
+ :- LogicalJoin(condition=[=($1, $9)], joinType=[left])
: :- LogicalJoin(condition=[true], joinType=[inner])
- : : :- LogicalJoin(condition=[true], joinType=[inner])
- : : : :- LogicalTableScan(table=[[default_catalog, default_database, l]])
- : : : +- LogicalAggregate(group=[{}], c=[COUNT()], ck=[COUNT($0, $1)])
- : : : +- LogicalProject(d=[$0], e=[$1])
- : : : +- LogicalTableScan(table=[[default_catalog, default_database, r]])
+ : : :- LogicalJoin(condition=[=($0, $5)], joinType=[left])
+ : : : :- LogicalJoin(condition=[true], joinType=[inner])
+ : : : : :- LogicalTableScan(table=[[default_catalog, default_database, l]])
+ : : : : +- LogicalAggregate(group=[{}], c=[COUNT()], ck=[COUNT() FILTER $0])
+ : : : : +- LogicalProject($f2=[OR(IS NOT NULL($0), IS NOT NULL($1))])
+ : : : : +- LogicalTableScan(table=[[default_catalog, default_database, r]])
+ : : : +- LogicalProject(i1=[$0], $f1=[true])
+ : : : +- LogicalAggregate(group=[{0}])
+ : : : +- LogicalProject(i1=[$0])
+ : : : +- LogicalFilter(condition=[IS NOT NULL($0)])
+ : : : +- LogicalTableScan(table=[[default_catalog, default_database, t]])
: : +- LogicalAggregate(group=[{}], c=[COUNT()], ck=[COUNT($0)])
: : +- LogicalProject(m=[$1])
: : +- LogicalTableScan(table=[[default_catalog, default_database, t2]])
@@ -917,17 +931,10 @@ LogicalProject(c=[$2])
: +- LogicalAggregate(group=[{0}])
: +- LogicalProject(m=[$1])
: +- LogicalTableScan(table=[[default_catalog, default_database, t2]])
- +- LogicalJoin(condition=[=($0, $3)], joinType=[left])
- :- LogicalFilter(condition=[=(2, $0)])
- : +- LogicalProject(d=[$0], e=[$1], $f2=[true])
- : +- LogicalAggregate(group=[{0, 1}])
- : +- LogicalProject(d=[$0], e=[$1])
- : +- LogicalTableScan(table=[[default_catalog, default_database, r]])
- +- LogicalProject(i1=[$0], $f1=[true])
- +- LogicalAggregate(group=[{0}])
- +- LogicalProject(i1=[$0])
- +- LogicalFilter(condition=[IS NOT NULL($0)])
- +- LogicalTableScan(table=[[default_catalog, default_database, t]])
+ +- LogicalProject(d=[$0], e=[$1], $f2=[true])
+ +- LogicalAggregate(group=[{0, 1}])
+ +- LogicalProject(d=[$0], e=[$1])
+ +- LogicalTableScan(table=[[default_catalog, default_database, r]])
]]>
</Resource>
</TestCase>
@@ -2182,16 +2189,24 @@ LogicalProject(b=[$1])
+- LogicalFilter(condition=[OR(=($3, 0), AND(IS NULL($12), >=($4, $3)))])
+- LogicalJoin(condition=[=(CASE(OR(=($6, 0), IS NOT TRUE(OR(IS NULL($0), IS NOT NULL($10), <($7, $6)))), 1, 2), $11)], joinType=[left])
:- LogicalJoin(condition=[=($0, $9)], joinType=[left])
- : :- LogicalJoin(condition=[=($0, $5)], joinType=[left])
+ : :- LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $5)], joinType=[left])
: : :- LogicalJoin(condition=[true], joinType=[inner])
: : : :- LogicalTableScan(table=[[default_catalog, default_database, l]])
: : : +- LogicalAggregate(group=[{}], c=[COUNT()], ck=[COUNT($0)])
: : : +- LogicalProject(d=[$0])
: : : +- LogicalTableScan(table=[[default_catalog, default_database, r]])
- : : +- LogicalAggregate(group=[{0}], c=[COUNT()], ck=[COUNT($1)])
- : : +- LogicalProject(i1=[$0], i=[$0])
- : : +- LogicalFilter(condition=[IS NOT NULL($0)])
- : : +- LogicalTableScan(table=[[default_catalog, default_database, t1]])
+ : : +- LogicalProject(i1=[$0], c=[CASE(IS NOT NULL($2), $2, 0)], ck=[CASE(IS NOT NULL($3), $3, 0)])
+ : : +- LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])
+ : : :- LogicalAggregate(group=[{0}])
+ : : : +- LogicalJoin(condition=[true], joinType=[inner])
+ : : : :- LogicalTableScan(table=[[default_catalog, default_database, l]])
+ : : : +- LogicalAggregate(group=[{}], c=[COUNT()], ck=[COUNT($0)])
+ : : : +- LogicalProject(d=[$0])
+ : : : +- LogicalTableScan(table=[[default_catalog, default_database, r]])
+ : : +- LogicalAggregate(group=[{0}], c=[COUNT()], ck=[COUNT($1)])
+ : : +- LogicalProject(i1=[$0], i=[$0])
+ : : +- LogicalFilter(condition=[IS NOT NULL($0)])
+ : : +- LogicalTableScan(table=[[default_catalog, default_database, t1]])
: +- LogicalFilter(condition=[=($1, $0)])
: +- LogicalProject(i=[$0], i2=[$1], $f2=[true])
: +- LogicalAggregate(group=[{0, 1}])
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQuerySemiJoinTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQuerySemiJoinTest.xml
index 8d81faec9dd..7f410f0ea78 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQuerySemiJoinTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQuerySemiJoinTest.xml
@@ -35,7 +35,7 @@ LogicalFilter(condition=[=($1, $cor0.b)])
<Resource name="optimized rel plan">
<![CDATA[
LogicalProject(s=[$1])
-+- LogicalJoin(condition=[=($2, $1)], joinType=[semi])
++- LogicalJoin(condition=[=($2, $0)], joinType=[semi])
:- LogicalAggregate(group=[{0}], s=[SUM($1)])
: +- LogicalProject(b=[$1], a=[$0])
: +- LogicalTableScan(table=[[default_catalog, default_database, x]])
@@ -1546,10 +1546,16 @@ LogicalProject(d=[$0], e=[$1])
<Resource name="optimized rel plan">
<![CDATA[
LogicalProject(c=[$2])
-+- LogicalJoin(condition=[AND(=(CASE(IS NOT NULL($2), 1, 2), $7), =(CASE(AND(<>($3, 0), IS NOT NULL($6), IS NOT NULL($1)), 3, 4), $8))], joinType=[inner])
- :- LogicalJoin(condition=[=($1, $5)], joinType=[left])
++- LogicalJoin(condition=[AND(=(CASE(IS NOT NULL($4), 1, 2), $9), =(CASE(AND(<>($5, 0), IS NOT NULL($8), IS NOT NULL($1)), 3, 4), $10))], joinType=[inner])
+ :- LogicalJoin(condition=[=($1, $7)], joinType=[left])
: :- LogicalJoin(condition=[true], joinType=[inner])
- : : :- LogicalTableScan(table=[[default_catalog, default_database, l]])
+ : : :- LogicalJoin(condition=[=($0, $3)], joinType=[left])
+ : : : :- LogicalTableScan(table=[[default_catalog, default_database, l]])
+ : : : +- LogicalProject(i1=[$0], $f1=[true])
+ : : : +- LogicalAggregate(group=[{0}])
+ : : : +- LogicalProject(i1=[$0])
+ : : : +- LogicalFilter(condition=[IS NOT NULL($0)])
+ : : : +- LogicalTableScan(table=[[default_catalog, default_database, t]])
: : +- LogicalAggregate(group=[{}], c=[COUNT()], ck=[COUNT($0)])
: : +- LogicalProject(m=[$1])
: : +- LogicalTableScan(table=[[default_catalog, default_database, t2]])
@@ -1557,15 +1563,9 @@ LogicalProject(c=[$2])
: +- LogicalAggregate(group=[{0}])
: +- LogicalProject(m=[$1])
: +- LogicalTableScan(table=[[default_catalog, default_database, t2]])
- +- LogicalJoin(condition=[=($0, $2)], joinType=[left])
- :- LogicalAggregate(group=[{0, 1}])
- : +- LogicalProject(d=[$0], e=[$1])
- : +- LogicalTableScan(table=[[default_catalog, default_database, r]])
- +- LogicalProject(i1=[$0], $f1=[true])
- +- LogicalAggregate(group=[{0}])
- +- LogicalProject(i1=[$0])
- +- LogicalFilter(condition=[IS NOT NULL($0)])
- +- LogicalTableScan(table=[[default_catalog, default_database, t]])
+ +- LogicalAggregate(group=[{0, 1}])
+ +- LogicalProject(d=[$0], e=[$1])
+ +- LogicalTableScan(table=[[default_catalog, default_database, r]])
]]>
</Resource>
</TestCase>
@@ -1612,7 +1612,7 @@ LogicalProject(d=[$1])
<![CDATA[
LogicalProject(s=[$1])
+- LogicalProject(b=[$0], s=[$1], $f2=[$2])
- +- LogicalJoin(condition=[AND(=($3, $4), =($4, $1))], joinType=[semi])
+ +- LogicalJoin(condition=[AND(=($3, $4), =($4, $0))], joinType=[semi])
:- LogicalProject(b=[$0], s=[$1], $f2=[$2], $f20=[CAST($2):BIGINT])
: +- LogicalAggregate(group=[{0}], s=[SUM($1)], agg#1=[MAX($1)])
: +- LogicalProject(b=[$1], a=[$0])
@@ -3832,12 +3832,16 @@ LogicalProject(d=[$0])
LogicalProject(b=[$1])
+- LogicalJoin(condition=[=(CASE(AND(IS NOT TRUE(OR(IS NULL($0), =($4, 0))), IS NOT NULL($8)), 1, 2), $9)], joinType=[inner])
:- LogicalJoin(condition=[=($0, $7)], joinType=[left])
- : :- LogicalJoin(condition=[=($0, $3)], joinType=[left])
+ : :- LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $3)], joinType=[left])
: : :- LogicalTableScan(table=[[default_catalog, default_database, l]])
- : : +- LogicalAggregate(group=[{0}], c=[COUNT()], ck=[COUNT($1)])
- : : +- LogicalProject(i1=[$0], i=[$0])
- : : +- LogicalFilter(condition=[IS NOT NULL($0)])
- : : +- LogicalTableScan(table=[[default_catalog, default_database, t1]])
+ : : +- LogicalProject(i1=[$0], c=[CASE(IS NOT NULL($2), $2, 0)], ck=[CASE(IS NOT NULL($3), $3, 0)])
+ : : +- LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left])
+ : : :- LogicalAggregate(group=[{0}])
+ : : : +- LogicalTableScan(table=[[default_catalog, default_database, l]])
+ : : +- LogicalAggregate(group=[{0}], c=[COUNT()], ck=[COUNT($1)])
+ : : +- LogicalProject(i1=[$0], i=[$0])
+ : : +- LogicalFilter(condition=[IS NOT NULL($0)])
+ : : +- LogicalTableScan(table=[[default_catalog, default_database, t1]])
: +- LogicalFilter(condition=[=($1, $0)])
: +- LogicalProject(i=[$0], i2=[$1], $f2=[true])
: +- LogicalAggregate(group=[{0, 1}])
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/InsertIntoValuesTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/InsertIntoValuesTest.xml
index 716fe8fa273..b4cf3cbfa47 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/InsertIntoValuesTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/InsertIntoValuesTest.xml
@@ -24,11 +24,11 @@ limitations under the License.
<![CDATA[
LogicalSink(table=[default_catalog.default_database.t1], fields=[EXPR$0, EXPR$1])
+- LogicalUnion(all=[true])
- :- LogicalProject(EXPR$0=[MAP(_UTF-16LE'a', _UTF-16LE'123':VARCHAR(6) CHARACTER SET "UTF-16LE", _UTF-16LE'b', _UTF-16LE'123456':VARCHAR(6) CHARACTER SET "UTF-16LE")], EXPR$1=[MAP(_UTF-16LE'k1', X'c0ffee':VARBINARY(3), _UTF-16LE'k2', X'babe':VARBINARY(3))])
+ :- LogicalProject(EXPR$0=[CAST(MAP(_UTF-16LE'a', _UTF-16LE'123':VARCHAR(6) CHARACTER SET "UTF-16LE", _UTF-16LE'b', _UTF-16LE'123456':VARCHAR(6) CHARACTER SET "UTF-16LE")):(VARCHAR(2147483647) CHARACTER SET "UTF-16LE", VARCHAR(2147483647) CHARACTER SET "UTF-16LE") MAP], EXPR$1=[CAST(MAP(_UTF-16LE'k1', X'c0ffee':VARBINARY(3), _UTF-16LE'k2', X'babe':VARBINARY(3))):(VARCHAR(2147483647) CHARACTER SET "UTF-16LE", VARBINARY(2147483647)) MAP])
: +- LogicalValues(tuples=[[{ 0 }]])
:- LogicalProject(EXPR$0=[null:(VARCHAR(2147483647) CHARACTER SET "UTF-16LE", VARCHAR(2147483647) CHARACTER SET "UTF-16LE") MAP], EXPR$1=[null:(VARCHAR(2147483647) CHARACTER SET "UTF-16LE", VARBINARY(2147483647)) MAP])
: +- LogicalValues(tuples=[[{ 0 }]])
- +- LogicalProject(EXPR$0=[MAP(_UTF-16LE'a', _UTF-16LE'1', _UTF-16LE'b', _UTF-16LE'1')], EXPR$1=[MAP(_UTF-16LE'k1', X'10':BINARY(1), _UTF-16LE'k2', X'20':BINARY(1))])
+ +- LogicalProject(EXPR$0=[CAST(MAP(_UTF-16LE'a', _UTF-16LE'1', _UTF-16LE'b', _UTF-16LE'1')):(VARCHAR(2147483647) CHARACTER SET "UTF-16LE", VARCHAR(2147483647) CHARACTER SET "UTF-16LE") MAP], EXPR$1=[CAST(MAP(_UTF-16LE'k1', X'10':BINARY(1), _UTF-16LE'k2', X'20':BINARY(1))):(VARCHAR(2147483647) CHARACTER SET "UTF-16LE", VARBINARY(2147483647)) MAP])
+- LogicalValues(tuples=[[{ 0 }]])
]]>
</Resource>
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/ValuesTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/ValuesTest.xml
index 42260c1b33f..36be34b657e 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/ValuesTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/ValuesTest.xml
@@ -25,9 +25,9 @@ limitations under the License.
LogicalProject(a=[$0], b=[$1]), rowType=[RecordType(INTEGER a, DECIMAL(20, 1) b)]
+- LogicalProject(a=[$0], b=[$1]), rowType=[RecordType(INTEGER a, DECIMAL(20, 1) b)]
+- LogicalUnion(all=[true]), rowType=[RecordType(INTEGER EXPR$0, DECIMAL(20, 1) EXPR$1)]
- :- LogicalProject(EXPR$0=[1], EXPR$1=[2.0:DECIMAL(2, 1)]), rowType=[RecordType(INTEGER EXPR$0, DECIMAL(2, 1) EXPR$1)]
+ :- LogicalProject(EXPR$0=[1], EXPR$1=[2.0:DECIMAL(20, 1)]), rowType=[RecordType(INTEGER EXPR$0, DECIMAL(20, 1) EXPR$1)]
: +- LogicalValues(tuples=[[{ 0 }]]), rowType=[RecordType(INTEGER ZERO)]
- +- LogicalProject(EXPR$0=[3], EXPR$1=[4:BIGINT]), rowType=[RecordType(INTEGER EXPR$0, BIGINT EXPR$1)]
+ +- LogicalProject(EXPR$0=[3], EXPR$1=[4:DECIMAL(20, 1)]), rowType=[RecordType(INTEGER EXPR$0, DECIMAL(20, 1) EXPR$1)]
+- LogicalValues(tuples=[[{ 0 }]]), rowType=[RecordType(INTEGER ZERO)]
]]>
</Resource>
@@ -36,7 +36,7 @@ LogicalProject(a=[$0], b=[$1]), rowType=[RecordType(INTEGER a, DECIMAL(20, 1) b)
Union(all=[true], union=[EXPR$0, EXPR$1]), rowType=[RecordType(INTEGER EXPR$0, DECIMAL(20, 1) EXPR$1)]
:- Calc(select=[1 AS EXPR$0, 2.0 AS EXPR$1]), rowType=[RecordType(INTEGER EXPR$0, DECIMAL(20, 1) EXPR$1)]
: +- Values(tuples=[[{ 0 }]]), rowType=[RecordType(INTEGER ZERO)]
-+- Calc(select=[3 AS EXPR$0, 4.0 AS EXPR$1]), rowType=[RecordType(INTEGER EXPR$0, DECIMAL(20, 1) EXPR$1)]
++- Calc(select=[3 AS EXPR$0, 4 AS EXPR$1]), rowType=[RecordType(INTEGER EXPR$0, DECIMAL(20, 1) EXPR$1)]
+- Values(tuples=[[{ 0 }]]), rowType=[RecordType(INTEGER ZERO)]
]]>
</Resource>
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/agg/GroupingSetsTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/agg/GroupingSetsTest.xml
index a5a504566af..a56cec43528 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/agg/GroupingSetsTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/agg/GroupingSetsTest.xml
@@ -159,6 +159,42 @@ Calc(select=[deptno, c])
+- Expand(projects=[{deptno, 0 AS $e}, {null AS deptno, 1 AS $e}])
+- Calc(select=[deptno])
+- TableSourceScan(table=[[default_catalog, default_database, emps]], fields=[empno, name, deptno, gender, city, empid, age, slacker, manager, joinedat])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testCALCITE1824">
+ <Resource name="sql">
+ <![CDATA[
+SELECT deptno, GROUP_ID() AS g, COUNT(*) AS c
+FROM scott_emp GROUP BY GROUPING SETS (deptno, (), ())
+ ]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalUnion(all=[true])
+:- LogicalProject(deptno=[$0], g=[0:BIGINT], c=[$1])
+: +- LogicalAggregate(group=[{0}], groups=[[{0}, {}]], c=[COUNT()])
+: +- LogicalProject(deptno=[$7])
+: +- LogicalTableScan(table=[[default_catalog, default_database, scott_emp]])
++- LogicalProject(deptno=[null:INTEGER], g=[1:BIGINT], c=[$0])
+ +- LogicalAggregate(group=[{}], c=[COUNT()])
+ +- LogicalTableScan(table=[[default_catalog, default_database, scott_emp]])
+]]>
+ </Resource>
+ <Resource name="optimized exec plan">
+ <![CDATA[
+Union(all=[true], union=[deptno, g, c])
+:- Calc(select=[deptno, 0 AS g, c])
+: +- GroupAggregate(groupBy=[deptno, $e], select=[deptno, $e, COUNT(*) AS c])
+: +- Exchange(distribution=[hash[deptno, $e]])
+: +- Expand(projects=[{deptno, 0 AS $e}, {null AS deptno, 1 AS $e}])
+: +- Calc(select=[deptno])
+: +- TableSourceScan(table=[[default_catalog, default_database, scott_emp]], fields=[empno, ename, job, mgr, hiredate, sal, comm, deptno])(reuse_id=[1])
++- Calc(select=[null:INTEGER AS deptno, 1 AS g, c])
+ +- GroupAggregate(select=[COUNT(*) AS c])
+ +- Exchange(distribution=[single])
+ +- Calc(select=[0 AS $f0])
+ +- Reused(reference_id=[1])
]]>
</Resource>
</TestCase>
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdDistinctRowCountTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdDistinctRowCountTest.scala
index 4beb3b1802a..0c781ca357e 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdDistinctRowCountTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdDistinctRowCountTest.scala
@@ -105,7 +105,7 @@ class FlinkRelMdDistinctRowCountTest extends FlinkRelMdHandlerTestBase {
assertEquals(1.0, mq.getDistinctRowCount(logicalProject, ImmutableBitSet.of(8), null))
assertEquals(1.0, mq.getDistinctRowCount(logicalProject, ImmutableBitSet.of(9), null))
assertEquals(1.0, mq.getDistinctRowCount(logicalProject, ImmutableBitSet.of(10), null))
- assertEquals(17.13, mq.getDistinctRowCount(logicalProject, ImmutableBitSet.of(11), null), 1e-2)
+ assertEquals(20.0, mq.getDistinctRowCount(logicalProject, ImmutableBitSet.of(11), null), 1e-2)
// id > 10
val expr1 = relBuilder
@@ -124,7 +124,10 @@ class FlinkRelMdDistinctRowCountTest extends FlinkRelMdHandlerTestBase {
assertEquals(1.0, mq.getDistinctRowCount(logicalProject, ImmutableBitSet.of(8), expr1))
assertEquals(1.0, mq.getDistinctRowCount(logicalProject, ImmutableBitSet.of(9), expr1))
assertEquals(1.0, mq.getDistinctRowCount(logicalProject, ImmutableBitSet.of(10), expr1))
- assertEquals(17.13, mq.getDistinctRowCount(logicalProject, ImmutableBitSet.of(11), expr1), 1e-2)
+ assertEquals(
+ 16.464466094067262,
+ mq.getDistinctRowCount(logicalProject, ImmutableBitSet.of(11), expr1),
+ 1e-2)
// age > 15 and class = 5
val expr2 = relBuilder
@@ -146,7 +149,7 @@ class FlinkRelMdDistinctRowCountTest extends FlinkRelMdHandlerTestBase {
assertEquals(1.0, mq.getDistinctRowCount(logicalProject, ImmutableBitSet.of(8), expr2))
assertEquals(1.0, mq.getDistinctRowCount(logicalProject, ImmutableBitSet.of(9), expr2))
assertEquals(1.0, mq.getDistinctRowCount(logicalProject, ImmutableBitSet.of(10), expr2))
- assertEquals(17.13, mq.getDistinctRowCount(logicalProject, ImmutableBitSet.of(11), expr2), 1e-2)
+ assertEquals(1.0, mq.getDistinctRowCount(logicalProject, ImmutableBitSet.of(11), expr2), 1e-2)
assertEquals(1.0, mq.getDistinctRowCount(logicalProject, ImmutableBitSet.of(0, 1), expr2))
}
@@ -200,7 +203,10 @@ class FlinkRelMdDistinctRowCountTest extends FlinkRelMdHandlerTestBase {
assertEquals(1.0, mq.getDistinctRowCount(calc, ImmutableBitSet.of(8), null))
assertEquals(1.0, mq.getDistinctRowCount(calc, ImmutableBitSet.of(9), null))
assertEquals(1.0, mq.getDistinctRowCount(calc, ImmutableBitSet.of(10), null))
- assertEquals(11.22, mq.getDistinctRowCount(calc, ImmutableBitSet.of(11), null), 1e-2)
+ assertEquals(
+ 16.464466094067262,
+ mq.getDistinctRowCount(calc, ImmutableBitSet.of(11), null),
+ 1e-2)
// class = 5
relBuilder.push(calc)
@@ -217,7 +223,10 @@ class FlinkRelMdDistinctRowCountTest extends FlinkRelMdHandlerTestBase {
assertEquals(1.0, mq.getDistinctRowCount(calc, ImmutableBitSet.of(8), expr2))
assertEquals(1.0, mq.getDistinctRowCount(calc, ImmutableBitSet.of(9), expr2))
assertEquals(1.0, mq.getDistinctRowCount(calc, ImmutableBitSet.of(10), expr2))
- assertEquals(11.22, mq.getDistinctRowCount(calc, ImmutableBitSet.of(11), expr2), 1e-2)
+ assertEquals(
+ 10.257214207425065,
+ mq.getDistinctRowCount(calc, ImmutableBitSet.of(11), expr2),
+ 1e-2)
}
@Test
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdPopulationSizeTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdPopulationSizeTest.scala
index 8de36795609..12ad645c462 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdPopulationSizeTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdPopulationSizeTest.scala
@@ -77,7 +77,7 @@ class FlinkRelMdPopulationSizeTest extends FlinkRelMdHandlerTestBase {
assertEquals(1.0, mq.getPopulationSize(logicalProject, ImmutableBitSet.of(8)))
assertEquals(1.0, mq.getPopulationSize(logicalProject, ImmutableBitSet.of(9)))
assertEquals(1.0, mq.getPopulationSize(logicalProject, ImmutableBitSet.of(10)))
- assertEquals(16.43, mq.getPopulationSize(logicalProject, ImmutableBitSet.of(11)), 1e-2)
+ assertEquals(20.0, mq.getPopulationSize(logicalProject, ImmutableBitSet.of(11)), 1e-2)
assertEquals(50.0, mq.getPopulationSize(logicalProject, ImmutableBitSet.of(0, 1)))
assertEquals(31.24, mq.getPopulationSize(logicalProject, ImmutableBitSet.of(1, 8)), 1e-2)
@@ -113,7 +113,7 @@ class FlinkRelMdPopulationSizeTest extends FlinkRelMdHandlerTestBase {
assertEquals(1.0, mq.getPopulationSize(logicalCalc, ImmutableBitSet.of(8)))
assertEquals(1.0, mq.getPopulationSize(logicalCalc, ImmutableBitSet.of(9)))
assertEquals(1.0, mq.getPopulationSize(logicalCalc, ImmutableBitSet.of(10)))
- assertEquals(11.22, mq.getPopulationSize(logicalCalc, ImmutableBitSet.of(11)), 1e-2)
+ assertEquals(20.0, mq.getPopulationSize(logicalCalc, ImmutableBitSet.of(11)), 1e-2)
assertEquals(50.0, mq.getPopulationSize(logicalCalc, ImmutableBitSet.of(0, 1)))
assertEquals(19.64, mq.getPopulationSize(logicalCalc, ImmutableBitSet.of(1, 8)), 1e-2)
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/GroupingSetsTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/GroupingSetsTest.scala
index 6b5c8ff9ca8..5e68ef39dd4 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/GroupingSetsTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/GroupingSetsTest.scala
@@ -21,7 +21,7 @@ import org.apache.flink.table.api._
import org.apache.flink.table.planner.plan.utils.FlinkRelOptUtil
import org.apache.flink.table.planner.utils.{TableTestBase, TableTestUtil}
-import org.assertj.core.api.Assertions.{assertThat, assertThatThrownBy}
+import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import java.sql.Date
@@ -407,9 +407,7 @@ class GroupingSetsTest extends TableTestBase {
|SELECT deptno, GROUP_ID() AS g, COUNT(*) AS c
|FROM scott_emp GROUP BY GROUPING SETS (deptno, (), ())
""".stripMargin
- assertThatThrownBy(() => util.verifyExecPlan(sqlQuery))
- .hasMessageContaining("GROUPING SETS are currently not supported")
- .isInstanceOf[TableException]
+ util.verifyExecPlan(sqlQuery)
}
@Test
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/agg/GroupingSetsITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/agg/GroupingSetsITCase.scala
index f79fb8ddbd2..d9aaae4e279 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/agg/GroupingSetsITCase.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/agg/GroupingSetsITCase.scala
@@ -618,13 +618,12 @@ class GroupingSetsITCase extends BatchTestBase {
"select deptno, group_id() as g, count(*) as c " +
"from scott_emp group by grouping sets (deptno, (), ())",
Seq(
+ // Suddenly before CALCITE-7126 it returned result different from Postgres/Oracle
row(10, 0, 3),
- row(10, 1, 3),
row(20, 0, 5),
- row(20, 1, 5),
row(30, 0, 6),
- row(30, 1, 6),
- row(null, 0, 14))
+ row(null, 0, 14),
+ row(null, 1, 14))
)
}
diff --git a/flink-table/pom.xml b/flink-table/pom.xml
index c17c122c0e6..b4d0ddd9654 100644
--- a/flink-table/pom.xml
+++ b/flink-table/pom.xml
@@ -79,7 +79,7 @@ under the License.
</dependencyManagement>
<properties>
- <calcite.version>1.40.0</calcite.version>
+ <calcite.version>1.41.0</calcite.version>
<janino.version>3.1.12</janino.version>
<guava.version>33.4.0-jre</guava.version>
<quartz.version>2.5.2</quartz.version>
(flink) 04/08: [FLINK-39859][table] Remove deprecated `JavaConversions` from `FlinkRelDistribution`
Posted by sn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
snuyanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 074f0f98444e23f33f8f69dc434d384446ab9d4c
Author: Sergey Nuyanzin <sn...@gmail.com>
AuthorDate: Sat Jul 4 08:43:14 2026 +0200
[FLINK-39859][table] Remove deprecated `JavaConversions` from `FlinkRelDistribution`
---
.../table/planner/plan/trait/FlinkRelDistribution.scala | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/trait/FlinkRelDistribution.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/trait/FlinkRelDistribution.scala
index 1e1e58953c7..59686ab8f09 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/trait/FlinkRelDistribution.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/trait/FlinkRelDistribution.scala
@@ -29,7 +29,7 @@ import org.apache.calcite.util.mapping.Mappings
import java.util
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
/**
* Description of the physical distribution of a relational expression. See [[RelDistribution]] for
@@ -98,7 +98,7 @@ class FlinkRelDistribution private (
override def apply(mapping: Mappings.TargetMapping): FlinkRelDistribution = {
if (distributionType == Type.HASH_DISTRIBUTED) {
val newKeys = new util.ArrayList[Integer]
- keys.foreach {
+ keys.asScala.foreach {
key =>
try {
val i = mapping.getTargetOpt(key)
@@ -114,7 +114,7 @@ class FlinkRelDistribution private (
FlinkRelDistribution.hash(newKeys, requireStrict)
} else if (distributionType == Type.RANGE_DISTRIBUTED) {
val newFieldCollations = new util.ArrayList[RelFieldCollation]
- fieldCollations.get.foreach {
+ fieldCollations.get.asScala.foreach {
fieldCollation =>
try {
val i = mapping.getTargetOpt(fieldCollation.getFieldIndex)
@@ -235,7 +235,7 @@ object FlinkRelDistribution {
/** Creates a range distribution. */
def range(collations: util.List[RelFieldCollation]): FlinkRelDistribution = {
- val keys = collations.map(i => Integer.valueOf(i.getFieldIndex)).toList
+ val keys = collations.asScala.map(i => Integer.valueOf(i.getFieldIndex)).toList
val fieldCollations = ImmutableList.copyOf[RelFieldCollation](collations)
canonize(
new FlinkRelDistribution(
@@ -244,12 +244,12 @@ object FlinkRelDistribution {
Some(fieldCollations)))
}
- def range(collations: RelFieldCollation*): FlinkRelDistribution = range(collations)
+ def range(collations: RelFieldCollation*): FlinkRelDistribution = range(collations.asJava)
def range(columns: util.Collection[_ <: Number]): FlinkRelDistribution = {
val keys = ImmutableIntList.copyOf(columns)
val collations = new util.ArrayList[RelFieldCollation]()
- columns.foreach(f => collations.add(FlinkRelOptUtil.ofRelFieldCollation(f.intValue())))
+ columns.asScala.foreach(f => collations.add(FlinkRelOptUtil.ofRelFieldCollation(f.intValue())))
val fieldCollations = ImmutableList.copyOf[RelFieldCollation](collations)
canonize(
new FlinkRelDistribution(RelDistribution.Type.RANGE_DISTRIBUTED, keys, Some(fieldCollations)))
(flink) 01/08: [FLINK-39859][table] Replace `Javaconverions` with `JavaConverters` in `RemoteCalcSplitRule` and `RemoteCalcSpliRule`
Posted by sn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
snuyanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit fb9c511f5901fe653dee080c58a3e0748b4a569d
Author: Sergey Nuyanzin <sn...@gmail.com>
AuthorDate: Mon Jun 29 10:28:12 2026 +0200
[FLINK-39859][table] Replace `Javaconverions` with `JavaConverters` in `RemoteCalcSplitRule` and `RemoteCalcSpliRule`
---
.../plan/rules/logical/RemoteCalcSplitRule.scala | 43 ++++++++---------
.../planner/plan/utils/RexNodeExtractor.scala | 56 ++++++++++++----------
2 files changed, 52 insertions(+), 47 deletions(-)
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcSplitRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcSplitRule.scala
index 26baa44f43d..f4537225dbc 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcSplitRule.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/RemoteCalcSplitRule.scala
@@ -28,7 +28,6 @@ import org.apache.calcite.sql.validate.SqlValidatorUtil
import java.util.function.Function
-import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
import scala.collection.mutable
@@ -94,8 +93,8 @@ abstract class RemoteCalcSplitRuleBase[T](
val bottomCalcProjects =
accessedFields.map(RexInputRef.of(_, input.getRowType)) ++ extractedRexNodes
val bottomCalcFieldNames = SqlValidatorUtil.uniquify(
- accessedFields.map(i => input.getRowType.getFieldNames.get(i)).toSeq ++
- extractedRexNodes.indices.map("f" + _),
+ (accessedFields.map(i => input.getRowType.getFieldNames.get(i)).toSeq ++
+ extractedRexNodes.indices.map("f" + _)).asJava,
rexBuilder.getTypeFactory.getTypeSystem.isSchemaCaseSensitive
)
@@ -105,7 +104,7 @@ abstract class RemoteCalcSplitRuleBase[T](
input,
RexProgram.create(
input.getRowType,
- bottomCalcProjects.toList,
+ bottomCalcProjects.toList.asJava,
splitComponents.bottomCalcCondition.orNull,
bottomCalcFieldNames,
rexBuilder))
@@ -120,7 +119,7 @@ abstract class RemoteCalcSplitRuleBase[T](
bottomCalc,
RexProgram.create(
bottomCalc.getRowType,
- topCalcProjects.map(_.accept(inputRewriter)),
+ topCalcProjects.map(_.accept(inputRewriter)).asJava,
splitComponents.topCalcCondition.map(_.accept(inputRewriter)).orNull,
calc.getRowType,
rexBuilder
@@ -204,7 +203,7 @@ class RemoteCalcSplitConditionRule(callFinder: RemoteCallFinder)
new SplitComponents(
None,
Option(program.getCondition).map(program.expandLocalRef(_).accept(splitter)),
- program.getProjectList.map(program.expandLocalRef))
+ program.getProjectList.asScala.toSeq.map(program.expandLocalRef))
}
}
@@ -217,7 +216,7 @@ abstract class RemoteCalcSplitProjectionRuleBase[T](
new SplitComponents(
Option(program.getCondition).map(program.expandLocalRef),
None,
- program.getProjectList.map(program.expandLocalRef(_).accept(splitter)))
+ program.getProjectList.asScala.toSeq.map(program.expandLocalRef(_).accept(splitter)))
}
}
@@ -240,7 +239,7 @@ abstract class RemoteCalcSplitRexFieldRuleBase(description: String, callFinder:
protected def containsFieldAccessAfterRemoteCall(node: RexNode): Boolean = {
node match {
- case call: RexCall => call.getOperands.exists(containsFieldAccessAfterRemoteCall)
+ case call: RexCall => call.getOperands.asScala.exists(containsFieldAccessAfterRemoteCall)
case x: RexFieldAccess => callFinder.containsRemoteCall(x.getReferenceExpr)
case _ => false
}
@@ -257,7 +256,7 @@ class RemoteCalcSplitProjectionRexFieldRule(callFinder: RemoteCallFinder)
override def matches(call: RelOptRuleCall): Boolean = {
val calc: FlinkLogicalCalc = call.rel(0).asInstanceOf[FlinkLogicalCalc]
- val projects = calc.getProgram.getProjectList.map(calc.getProgram.expandLocalRef)
+ val projects = calc.getProgram.getProjectList.asScala.toSeq.map(calc.getProgram.expandLocalRef)
projects.exists(containsFieldAccessAfterRemoteCall)
}
@@ -265,7 +264,7 @@ class RemoteCalcSplitProjectionRexFieldRule(callFinder: RemoteCallFinder)
new SplitComponents(
Option(program.getCondition).map(program.expandLocalRef),
None,
- program.getProjectList.map(_.accept(splitter)))
+ program.getProjectList.asScala.toSeq.map(_.accept(splitter)))
}
}
@@ -288,7 +287,7 @@ class RemoteCalcSplitConditionRexFieldRule(callFinder: RemoteCallFinder)
new SplitComponents(
None,
Option(program.getCondition).map(_.accept(splitter)),
- program.getProjectList.map(_.accept(splitter)))
+ program.getProjectList.asScala.toSeq.map(_.accept(splitter)))
}
}
@@ -302,7 +301,7 @@ class RemoteCalcSplitProjectionRule(callFinder: RemoteCallFinder)
override def matches(call: RelOptRuleCall): Boolean = {
val calc: FlinkLogicalCalc = call.rel(0).asInstanceOf[FlinkLogicalCalc]
- val projects = calc.getProgram.getProjectList.map(calc.getProgram.expandLocalRef)
+ val projects = calc.getProgram.getProjectList.asScala.toSeq.map(calc.getProgram.expandLocalRef)
// matches if it contains both Remote functions and Java functions in the projection
projects.exists(callFinder.containsRemoteCall) && projects.exists(
@@ -313,7 +312,7 @@ class RemoteCalcSplitProjectionRule(callFinder: RemoteCallFinder)
program: RexProgram,
node: RexNode,
matchState: Option[Nothing]): Boolean = {
- program.getProjectList
+ program.getProjectList.asScala.toSeq
.map(program.expandLocalRef)
.exists(callFinder.isNonRemoteCall) == callFinder.isRemoteCall(node)
}
@@ -328,7 +327,7 @@ class RemoteCalcExpandProjectRule(callFinder: RemoteCallFinder)
override def matches(call: RelOptRuleCall): Boolean = {
val calc: FlinkLogicalCalc = call.rel(0).asInstanceOf[FlinkLogicalCalc]
- val projects = calc.getProgram.getProjectList.map(calc.getProgram.expandLocalRef)
+ val projects = calc.getProgram.getProjectList.asScala.toSeq.map(calc.getProgram.expandLocalRef)
projects.exists(callFinder.containsRemoteCall) && projects.exists(containsFieldAccessInputs)
}
@@ -343,12 +342,12 @@ class RemoteCalcExpandProjectRule(callFinder: RemoteCallFinder)
new SplitComponents(
Option(program.getCondition).map(program.expandLocalRef),
None,
- program.getProjectList.map(program.expandLocalRef(_).accept(splitter)))
+ program.getProjectList.asScala.toSeq.map(program.expandLocalRef(_).accept(splitter)))
}
private def containsFieldAccessInputs(node: RexNode): Boolean = {
node match {
- case call: RexCall => call.getOperands.exists(containsFieldAccessInputs)
+ case call: RexCall => call.getOperands.asScala.exists(containsFieldAccessInputs)
case _: RexFieldAccess => true
case _ => false
}
@@ -364,7 +363,7 @@ class RemoteCalcPushConditionRule(callFinder: RemoteCallFinder)
override def matches(call: RelOptRuleCall): Boolean = {
val calc: FlinkLogicalCalc = call.rel(0).asInstanceOf[FlinkLogicalCalc]
- val projects = calc.getProgram.getProjectList.map(calc.getProgram.expandLocalRef)
+ val projects = calc.getProgram.getProjectList.asScala.toSeq.map(calc.getProgram.expandLocalRef)
// matches if all the following conditions hold true:
// 1) the condition is not null
@@ -382,7 +381,7 @@ class RemoteCalcPushConditionRule(callFinder: RemoteCallFinder)
new SplitComponents(
Option(program.getCondition).map(program.expandLocalRef),
None,
- program.getProjectList.map(program.expandLocalRef))
+ program.getProjectList.asScala.toSeq.map(program.expandLocalRef))
}
}
@@ -397,7 +396,7 @@ class RemoteCalcRewriteProjectionRule(callFinder: RemoteCallFinder)
override def matches(call: RelOptRuleCall): Boolean = {
val calc: FlinkLogicalCalc = call.rel(0).asInstanceOf[FlinkLogicalCalc]
- val projects = calc.getProgram.getProjectList.map(calc.getProgram.expandLocalRef)
+ val projects = calc.getProgram.getProjectList.asScala.toSeq.map(calc.getProgram.expandLocalRef)
// matches if all the following conditions hold true:
// 1) it contains Remote functions in the projection
@@ -419,7 +418,7 @@ class RemoteCalcRewriteProjectionRule(callFinder: RemoteCallFinder)
new SplitComponents(
None,
None,
- program.getProjectList.map(program.expandLocalRef(_).accept(splitter)))
+ program.getProjectList.asScala.toSeq.map(program.expandLocalRef(_).accept(splitter)))
}
}
@@ -440,7 +439,7 @@ class ScalarFunctionSplitter(
if (needConvert(call)) {
getExtractedRexNode(call)
} else {
- call.clone(call.getType, call.getOperands.asScala.map(_.accept(this)))
+ call.clone(call.getType, call.getOperands.asScala.map(_.accept(this)).asJava)
}
}
@@ -536,7 +535,7 @@ private class ExtractedFunctionInputRewriter(
}
override def visitCall(call: RexCall): RexNode = {
- call.clone(call.getType, call.getOperands.asScala.map(_.accept(this)))
+ call.clone(call.getType, call.getOperands.asScala.map(_.accept(this)).asJava)
}
override def visitFieldAccess(fieldAccess: RexFieldAccess): RexNode = {
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/RexNodeExtractor.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/RexNodeExtractor.scala
index 4e2d1d66205..59c1ae9e2e3 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/RexNodeExtractor.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/RexNodeExtractor.scala
@@ -48,7 +48,6 @@ import java.time.{ZoneId, ZoneOffset}
import java.util
import java.util.{Collections, List => JList}
-import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
@@ -67,7 +66,7 @@ object RexNodeExtractor extends Logging {
def extractRefInputFields(exprs: JList[RexNode]): Array[Int] = {
val visitor = new InputRefVisitor
// extract referenced input fields from expressions
- exprs.foreach(_.accept(visitor))
+ exprs.asScala.foreach(_.accept(visitor))
visitor.getFields
}
@@ -86,7 +85,7 @@ object RexNodeExtractor extends Logging {
exprs: JList[RexNode],
usedFields: Array[Int]): Array[Array[JList[String]]] = {
val visitor = new RefFieldAccessorVisitor(usedFields)
- exprs.foreach(_.accept(visitor))
+ exprs.asScala.foreach(_.accept(visitor))
visitor.getProjectedFields
}
@@ -168,8 +167,9 @@ object RexNodeExtractor extends Logging {
partitionFieldNames: Array[String]): (RexNode, RexNode) = {
val (partitionPredicates, nonPartitionPredicates) =
extractPartitionPredicateList(expr, inputFieldNames, rexBuilder, partitionFieldNames)
- val partitionPredicate = RexUtil.composeConjunction(rexBuilder, partitionPredicates)
- val nonPartitionPredicate = RexUtil.composeConjunction(rexBuilder, nonPartitionPredicates)
+ val partitionPredicate = RexUtil.composeConjunction(rexBuilder, partitionPredicates.asJava)
+ val nonPartitionPredicate =
+ RexUtil.composeConjunction(rexBuilder, nonPartitionPredicates.asJava)
(partitionPredicate, nonPartitionPredicate)
}
@@ -199,7 +199,8 @@ object RexNodeExtractor extends Logging {
val conjunctions = RelOptUtil.conjunctions(cnf)
val (partitionPredicates, nonPartitionPredicates) =
- conjunctions.partition(isSupportedPartitionPredicate(_, partitionFieldNames, inputFieldNames))
+ conjunctions.asScala.toSeq.partition(
+ isSupportedPartitionPredicate(_, partitionFieldNames, inputFieldNames))
(partitionPredicates, nonPartitionPredicates)
}
@@ -282,7 +283,7 @@ class InputRefVisitor extends RexVisitorImpl[Unit](true) {
fields += inputRef.getIndex
override def visitCall(call: RexCall): Unit =
- call.operands.foreach(operand => operand.accept(this))
+ call.operands.asScala.foreach(operand => operand.accept(this))
}
/** A RexVisitor to extract used nested input fields */
@@ -294,10 +295,10 @@ class RefFieldAccessorVisitor(usedFields: Array[Int]) extends RexVisitorImpl[Uni
private val order: Map[Int, Int] = usedFields.zipWithIndex.toMap
private def isPrefix(left: JList[String], right: JList[String]): Boolean = {
- if (right.length < left.length) {
+ if (right.size < left.size) {
false
} else {
- right.take(left.length).zip(left).foldLeft(true) {
+ right.asScala.take(left.size).zip(left.asScala).foldLeft(true) {
case (ans, (lName, rName)) => {
if (ans) {
lName.equals(rName)
@@ -319,20 +320,21 @@ class RefFieldAccessorVisitor(usedFields: Array[Int]) extends RexVisitorImpl[Uni
// get prefix field accesses
val prefixAccesses = sorted.foldLeft(Nil: List[JList[String]]) {
(prefixAccesses, nestedAccess) =>
+ val nestedAccessJava: JList[String] = nestedAccess.asJava
prefixAccesses match {
// first access => add access
- case Nil => List[JList[String]](nestedAccess)
+ case Nil => List[JList[String]](nestedAccessJava)
// top-level access already found => return top-level access
case head :: Nil if head.get(0).equals("*") => prefixAccesses
// access is top-level access => return top-level access
- case _ :: _ if nestedAccess.get(0).equals("*") => List(util.Arrays.asList("*"))
+ case _ :: _ if nestedAccessJava.get(0).equals("*") => List(util.Arrays.asList("*"))
case _ =>
- if (isPrefix(prefixAccesses.head, nestedAccess)) {
+ if (isPrefix(prefixAccesses.head, nestedAccessJava)) {
// previous access is a prefix of this access => do not add access
prefixAccesses
} else {
// previous access is not prefix of this access => add access
- nestedAccess :: prefixAccesses
+ nestedAccessJava :: prefixAccesses
}
}
}
@@ -370,7 +372,7 @@ class RefFieldAccessorVisitor(usedFields: Array[Int]) extends RexVisitorImpl[Uni
}
override def visitCall(call: RexCall): Unit =
- call.operands.foreach(operand => operand.accept(this))
+ call.operands.asScala.foreach(operand => operand.accept(this))
}
/**
@@ -505,7 +507,7 @@ class RexNodeToExpressionConverter(
override def visitCall(oriRexCall: RexCall): Option[ResolvedExpression] = {
val rexCall = FlinkRexUtil.expandSearch(rexBuilder, oriRexCall).asInstanceOf[RexCall]
- val operands = rexCall.getOperands.map(operand => operand.accept(this).orNull)
+ val operands = rexCall.getOperands.asScala.toSeq.map(operand => operand.accept(this).orNull)
val outputType = fromLogicalTypeToDataType(FlinkTypeFactory.toLogicalType(rexCall.getType))
@@ -515,17 +517,21 @@ class RexNodeToExpressionConverter(
} else {
rexCall.getOperator match {
case SqlStdOperatorTable.OR =>
- Option(operands.reduceLeft((l, r) => CallExpression.permanent(OR, Seq(l, r), outputType)))
+ Option(
+ operands.reduceLeft(
+ (l, r) => CallExpression.permanent(OR, Seq(l, r).asJava, outputType)))
case SqlStdOperatorTable.AND =>
Option(
- operands.reduceLeft((l, r) => CallExpression.permanent(AND, Seq(l, r), outputType)))
+ operands.reduceLeft(
+ (l, r) => CallExpression.permanent(AND, Seq(l, r).asJava, outputType)))
case SqlStdOperatorTable.CAST =>
Option(
- CallExpression.permanent(CAST, Seq(operands.head, typeLiteral(outputType)), outputType))
+ CallExpression
+ .permanent(CAST, Seq(operands.head, typeLiteral(outputType)).asJava, outputType))
case FlinkSqlOperatorTable.TRY_CAST =>
Option(
CallExpression
- .permanent(TRY_CAST, Seq(operands.head, typeLiteral(outputType)), outputType))
+ .permanent(TRY_CAST, Seq(operands.head, typeLiteral(outputType)).asJava, outputType))
case _: SqlFunction | _: SqlPostfixOperator =>
val names = new util.ArrayList[String](rexCall.getOperator.getNameAsId.names)
names.set(names.size() - 1, replace(names.get(names.size() - 1)))
@@ -551,15 +557,15 @@ class RexNodeToExpressionConverter(
case Some(dataType) =>
val schema = NestedProjectionUtil.build(Collections.singletonList(fieldAccess), dataType)
val fieldIndices = NestedProjectionUtil.convertToIndexArray(schema)
- var (topLevelColumnName, nestedColumn) = schema.columns.head
+ var (topLevelColumnName, nestedColumn) = schema.columns.asScala.head
val fieldNames = new ArrayBuffer[String]()
while (!nestedColumn.isLeaf) {
- fieldNames.add(topLevelColumnName)
- topLevelColumnName = nestedColumn.children.head._1
- nestedColumn = nestedColumn.children.head._2
+ fieldNames += topLevelColumnName
+ topLevelColumnName = nestedColumn.children.asScala.head._1
+ nestedColumn = nestedColumn.children.asScala.head._2
}
- fieldNames.add(topLevelColumnName)
+ fieldNames += topLevelColumnName
Some(
new NestedFieldReferenceExpression(
@@ -607,7 +613,7 @@ class RexNodeToExpressionConverter(
resolvedFunction.isTemporary,
identifier,
resolvedFunction.getDefinition,
- operands,
+ operands.asJava,
outputType)
)
} else {
(flink) 07/08: [FLINK-39859][table] Replace deprecated usage of `SqlLiteral#createTimestamp`
Posted by sn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
snuyanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit f1c2318be9c45ae470ee8f8b157d00932bc1b288
Author: Sergey Nuyanzin <sn...@gmail.com>
AuthorDate: Sat Jul 4 14:56:40 2026 +0200
[FLINK-39859][table] Replace deprecated usage of `SqlLiteral#createTimestamp`
---
.../apache/flink/table/planner/calcite/FlinkCalciteSqlValidator.java | 2 ++
1 file changed, 2 insertions(+)
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidator.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidator.java
index 39fe099152f..651a86925c3 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidator.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidator.java
@@ -67,6 +67,7 @@ import org.apache.calcite.sql.SqlWindowTableFunction;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.type.SqlOperandMetadata;
import org.apache.calcite.sql.type.SqlOperandTypeChecker;
+import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.type.SqlTypeUtil;
import org.apache.calcite.sql.validate.DelegatingScope;
import org.apache.calcite.sql.validate.IdentifierNamespace;
@@ -273,6 +274,7 @@ public final class FlinkCalciteSqlValidator extends FlinkSqlParsingValidator {
sqlSnapshot.setOperand(
1,
SqlLiteral.createTimestamp(
+ SqlTypeName.TIMESTAMP,
timestampString,
rexLiteral.getType().getPrecision(),
sqlSnapshot.getPeriod().getParserPosition()));